framework,version,device,op_name,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,1,128,1,float16,float16,0,71.33876546223958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,1,128,1,fp8,fp8,0,42.91260782877604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,1,128,1,float16,fp8,0,71.76063537597656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,2,128,1,float16,float16,0,71.89011637369792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,2,128,1,float16,fp8,0,72.3192647298177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,2,128,1,fp8,fp8,0,42.36563618977865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,4,128,1,float16,fp8,0,70.75140889485677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,4,128,1,float16,float16,0,71.63220723470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,1,128,1,float16,float16,0,33.08979797363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,4,128,1,fp8,fp8,0,42.45061747233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,8,128,1,float16,float16,0,73.15226745605469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,8,128,1,float16,fp8,0,71.37953694661458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,64,8,128,1,fp8,fp8,0,42.60443623860677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,64,128,1,float16,float16,0,33.08658091227213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,1,128,1,float16,fp8,0,32.89494323730469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,1,128,1,fp8,fp8,0,21.36963653564453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,64,128,1,fp8,fp8,0,20.92523193359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,64,128,1,float16,fp8,0,32.43256123860677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,2,128,1,float16,float16,0,33.485557556152344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,2,128,1,float16,fp8,0,32.656341552734375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,2,128,1,fp8,fp8,0,21.3590571085612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,4,128,1,float16,float16,0,32.91156768798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,4,128,1,float16,fp8,0,32.997510274251304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,4,128,1,fp8,fp8,0,21.501317342122395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,8,128,1,fp8,fp8,0,21.632469177246094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,8,128,1,float16,fp8,0,32.88089497884115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,1,128,1,float16,float16,0,16.589749654134113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,64,8,128,1,float16,float16,0,32.65568542480469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,64,128,1,float16,float16,0,16.279866536458332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,64,128,1,float16,fp8,0,16.337610880533855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,1,128,1,float16,fp8,0,16.542181650797527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,1,128,1,fp8,fp8,0,10.979456583658854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,2,128,1,float16,fp8,0,16.61135991414388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,2,128,1,fp8,fp8,0,10.721424102783203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,2,128,1,float16,float16,0,16.572533925374348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,64,128,1,fp8,fp8,0,10.947877248128256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,4,128,1,float16,float16,0,16.702757517496746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,4,128,1,fp8,fp8,0,10.717301686604818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,4,128,1,float16,fp8,0,16.554181416829426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,8,128,1,float16,float16,0,16.941360473632812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,8,128,1,float16,fp8,0,16.577749888102215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,64,8,128,1,fp8,fp8,0,10.837231953938803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,64,128,1,float16,float16,0,8.386117299397787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,1,128,1,float16,fp8,0,8.485722859700521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,1,128,1,float16,float16,0,8.463567733764648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,1,128,1,fp8,fp8,0,5.616207758585612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,64,128,1,float16,fp8,0,8.341925303141275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,64,128,1,fp8,fp8,0,5.516656239827474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,2,128,1,float16,float16,0,8.490853627522787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,2,128,1,float16,fp8,0,8.33303451538086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,2,128,1,fp8,fp8,0,5.398517608642578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,4,128,1,float16,float16,0,8.608432133992514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,4,128,1,float16,fp8,0,8.34880002339681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,4,128,1,fp8,fp8,0,5.652917226155599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,8,128,1,float16,float16,0,8.499338785807291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,8,128,1,float16,fp8,0,8.438735961914062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,64,8,128,1,fp8,fp8,0,5.43235715230306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,1,128,1,float16,float16,0,37.66522216796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,1,128,1,float16,fp8,0,37.868324279785156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,1,128,1,fp8,fp8,0,24.749913533528645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,2,128,1,float16,float16,0,38.166516621907554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,2,128,1,fp8,fp8,0,25.19109853108724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,2,128,1,float16,fp8,0,38.39385477701823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,4,128,1,float16,float16,0,38.20138041178385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,4,128,1,float16,fp8,0,37.73161061604818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,1,128,1,float16,float16,0,19.04549280802409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,4,128,1,fp8,fp8,0,24.71295928955078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,8,128,1,fp8,fp8,0,24.750831604003906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,8,128,1,float16,fp8,0,37.99944051106771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,64,8,128,1,float16,float16,0,38.208335876464844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,1,128,1,fp8,fp8,0,12.476256052652994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,1,128,1,float16,fp8,0,19.027781168619793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,64,128,1,float16,float16,0,19.267813364664715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,64,128,1,fp8,fp8,0,12.638837178548178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,64,128,1,float16,fp8,0,19.268090565999348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,2,128,1,float16,float16,0,19.137898763020832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,2,128,1,float16,fp8,0,19.00586700439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,2,128,1,fp8,fp8,0,12.51699193318685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,4,128,1,float16,float16,0,18.812362670898438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,4,128,1,fp8,fp8,0,12.56320571899414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,4,128,1,float16,fp8,0,19.30396270751953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,8,128,1,float16,float16,0,19.202037811279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,8,128,1,float16,fp8,0,19.49772771199544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,64,8,128,1,fp8,fp8,0,12.674991607666016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,1,128,1,float16,float16,0,9.687333424886068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,1,128,1,float16,fp8,0,9.504986445109049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,1,128,1,fp8,fp8,0,6.243584314982097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,64,128,1,float16,float16,0,9.60582415262858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,2,128,1,float16,float16,0,9.708159764607748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,64,128,1,float16,fp8,0,9.471871693929037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,2,128,1,float16,fp8,0,9.555760065714518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,64,128,1,fp8,fp8,0,6.334138870239258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,2,128,1,fp8,fp8,0,6.281034469604492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,4,128,1,float16,float16,0,9.594970703125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,4,128,1,fp8,fp8,0,6.388224283854167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,4,128,1,float16,fp8,0,9.668368021647135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,8,128,1,float16,float16,0,9.659077326456705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,8,128,1,fp8,fp8,0,6.22657585144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,64,8,128,1,float16,fp8,0,9.775903701782227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,1,128,1,float16,float16,0,4.91978136698405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,1,128,1,float16,fp8,0,4.8860212961832685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,64,128,1,float16,float16,0,4.768495877583821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,64,128,1,float16,fp8,0,4.678160031636556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,1,128,1,fp8,fp8,0,3.2713438669840493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,2,128,1,float16,float16,0,4.867242813110352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,64,128,1,fp8,fp8,0,3.245375951131185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,2,128,1,float16,fp8,0,4.6576798756917315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,2,128,1,fp8,fp8,0,3.252645174662272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,4,128,1,float16,float16,0,4.773296038309733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,4,128,1,float16,fp8,0,4.815386772155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,4,128,1,fp8,fp8,0,3.27130126953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,8,128,1,float16,fp8,0,4.76804796854655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,8,128,1,fp8,fp8,0,3.266026814778646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,64,8,128,1,float16,float16,0,4.977151870727539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,1,128,1,float16,float16,0,26.889785766601562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,1,128,1,fp8,fp8,0,17.581231435139973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,1,128,1,float16,fp8,0,26.98810577392578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,2,128,1,float16,float16,0,26.76788330078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,2,128,1,fp8,fp8,0,17.743845621744793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,2,128,1,float16,fp8,0,27.219685872395832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,4,128,1,float16,float16,0,27.009300231933594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,4,128,1,float16,fp8,0,27.338734944661457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,1,128,1,float16,float16,0,13.788965861002604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,4,128,1,fp8,fp8,0,17.859903971354168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,8,128,1,fp8,fp8,0,17.664559682210285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,8,128,1,float16,float16,0,27.06153615315755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,64,8,128,1,float16,fp8,0,27.02118428548177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,1,128,1,float16,fp8,0,13.739584604899088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,64,128,1,float16,float16,0,13.580219268798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,64,128,1,float16,fp8,0,13.744186401367188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,1,128,1,fp8,fp8,0,8.999167760213217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,2,128,1,float16,float16,0,13.42193603515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,2,128,1,fp8,fp8,0,9.037066777547201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,2,128,1,float16,fp8,0,13.37127431233724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,64,128,1,fp8,fp8,0,9.176730473836264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,4,128,1,float16,float16,0,13.757813771565756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,4,128,1,float16,fp8,0,13.498650868733725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,4,128,1,fp8,fp8,0,9.053904215494791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,8,128,1,float16,float16,0,13.45571772257487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,1,128,1,float16,float16,0,6.745440165201823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,8,128,1,float16,fp8,0,13.399093627929688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,64,8,128,1,fp8,fp8,0,8.879936218261719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,1,128,1,float16,fp8,0,6.803925196329753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,64,128,1,float16,float16,0,6.739706675211589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,1,128,1,fp8,fp8,0,4.406810760498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,2,128,1,float16,float16,0,6.731903711954753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,64,128,1,float16,fp8,0,6.786645253499349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,64,128,1,fp8,fp8,0,4.654842694600423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,2,128,1,float16,fp8,0,6.706719716389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,2,128,1,fp8,fp8,0,4.412320137023926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,4,128,1,float16,float16,0,6.921637217203776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,4,128,1,fp8,fp8,0,4.523237228393555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,4,128,1,float16,fp8,0,6.799552281697591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,8,128,1,float16,float16,0,6.585669199625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,8,128,1,fp8,fp8,0,4.566426595052083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,1,128,1,float16,float16,0,3.395397186279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,1,128,1,float16,fp8,0,3.4311841328938804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,64,128,1,float16,float16,0,3.332122802734375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,64,8,128,1,float16,fp8,0,6.77400016784668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,64,128,1,float16,fp8,0,3.318810780843099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,64,128,1,fp8,fp8,0,2.420805295308431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,1,128,1,fp8,fp8,0,2.3786773681640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,2,128,1,float16,fp8,0,3.3547999064127603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,2,128,1,fp8,fp8,0,2.378437360127767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,2,128,1,float16,float16,0,3.4473705291748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,4,128,1,float16,float16,0,3.4441121419270835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,4,128,1,fp8,fp8,0,2.3815414110819497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,8,128,1,float16,float16,0,3.418725331624349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,8,128,1,float16,fp8,0,3.3692426681518555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,4,128,1,float16,fp8,0,3.4194294611612954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,64,8,128,1,fp8,fp8,0,2.388805389404297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,1,128,1,float16,float16,0,35.163317362467446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,1,128,1,fp8,fp8,0,23.44097137451172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,1,128,1,float16,fp8,0,35.975077311197914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,2,128,1,float16,float16,0,35.62369028727213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,2,128,1,fp8,fp8,0,23.702112833658855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,2,128,1,float16,fp8,0,36.059295654296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,4,128,1,float16,fp8,0,35.30575052897135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,4,128,1,float16,float16,0,35.67648569742838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,1,128,1,float16,float16,0,17.712331136067707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,4,128,1,fp8,fp8,0,23.504005432128906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,8,128,1,float16,float16,0,35.23486836751302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,8,128,1,fp8,fp8,0,23.8495356241862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,64,8,128,1,float16,fp8,0,35.94230397542318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,64,128,1,float16,float16,0,17.715850830078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,64,128,1,float16,fp8,0,18.22543462117513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,1,128,1,float16,fp8,0,17.67131169637044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,1,128,1,fp8,fp8,0,11.818453470865885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,64,128,1,fp8,fp8,0,12.214436848958334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,2,128,1,float16,float16,0,17.565157572428387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,2,128,1,float16,fp8,0,17.836959838867188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,2,128,1,fp8,fp8,0,12.132837931315104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,4,128,1,float16,float16,0,17.71564229329427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,4,128,1,float16,fp8,0,18.051483154296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,4,128,1,fp8,fp8,0,11.888192494710287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,8,128,1,float16,float16,0,17.71624501546224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,8,128,1,float16,fp8,0,17.69171142578125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,64,8,128,1,fp8,fp8,0,11.884586334228516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,1,128,1,float16,float16,0,8.83631451924642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,1,128,1,float16,fp8,0,8.780965169270834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,1,128,1,fp8,fp8,0,5.806517283121745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,64,128,1,float16,float16,0,8.920570373535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,2,128,1,float16,float16,0,8.770314534505209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,2,128,1,fp8,fp8,0,5.929738362630208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,2,128,1,float16,fp8,0,8.696479797363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,64,128,1,float16,fp8,0,9.048639933268229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,64,128,1,fp8,fp8,0,6.082543690999349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,4,128,1,float16,float16,0,8.905413309733072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,4,128,1,float16,fp8,0,8.806362787882486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,4,128,1,fp8,fp8,0,5.990293502807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,8,128,1,float16,fp8,0,8.867162704467773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,8,128,1,float16,float16,0,8.866101582845053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,1,128,1,float16,float16,0,4.287055969238281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,64,8,128,1,fp8,fp8,0,5.837039947509766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,1,128,1,float16,fp8,0,4.2564748128255205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,64,128,1,float16,fp8,0,4.372474670410156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,64,128,1,float16,float16,0,4.511861483256022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,1,128,1,fp8,fp8,0,3.033050537109375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,64,128,1,fp8,fp8,0,3.070026715596517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,2,128,1,float16,float16,0,4.413536071777344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,2,128,1,float16,fp8,0,4.382143974304199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,2,128,1,fp8,fp8,0,3.000373204549154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,4,128,1,float16,float16,0,4.296304066975911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,4,128,1,float16,fp8,0,4.375882784525554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,4,128,1,fp8,fp8,0,3.0222721099853516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,8,128,1,float16,fp8,0,4.413109461466472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,8,128,1,float16,float16,0,4.511472066243489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,64,8,128,1,fp8,fp8,0,3.0179519653320312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,64,128,1,float16,float16,0,2.2725067138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,64,128,1,float16,fp8,0,2.239840030670166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,1,128,1,float16,float16,0,2.2984372774759927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,1,128,1,float16,fp8,0,2.2942773501078286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,1,128,1,fp8,fp8,0,1.6422026952107747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,2,128,1,float16,float16,0,2.309978644053141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,64,128,1,fp8,fp8,0,1.6589919726053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,2,128,1,float16,fp8,0,2.2626880009969077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,2,128,1,fp8,fp8,0,1.6484160423278809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,4,128,1,float16,fp8,0,2.270842711130778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,4,128,1,float16,float16,0,2.2891999880472818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,4,128,1,fp8,fp8,0,1.6499892870585124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,8,128,1,float16,float16,0,2.303258736928304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,8,128,1,fp8,fp8,0,1.650917371114095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,64,8,128,1,float16,fp8,0,2.278981367746989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,1,128,1,float16,float16,0,20.98072052001953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,1,128,1,fp8,fp8,0,14.133861541748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,1,128,1,float16,fp8,0,20.596832275390625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,2,128,1,fp8,fp8,0,14.154752095540365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,2,128,1,float16,float16,0,20.964864095052082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,2,128,1,float16,fp8,0,21.121082305908203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,4,128,1,float16,fp8,0,21.5786616007487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,4,128,1,float16,float16,0,20.80413818359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,1,128,1,float16,float16,0,10.523397445678711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,4,128,1,fp8,fp8,0,14.272064208984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,8,128,1,float16,fp8,0,20.75652821858724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,1,128,1,float16,fp8,0,10.337546666463217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,8,128,1,float16,float16,0,20.907376607259113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,64,8,128,1,fp8,fp8,0,14.297018686930338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,64,128,1,float16,float16,0,10.438506444295248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,64,128,1,fp8,fp8,0,7.682416280110677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,64,128,1,float16,fp8,0,10.641557057698568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,1,128,1,fp8,fp8,0,7.052527745564778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,2,128,1,fp8,fp8,0,7.052480061848958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,2,128,1,float16,fp8,0,10.368170420328775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,2,128,1,float16,float16,0,10.477338790893555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,4,128,1,float16,float16,0,10.488543828328451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,4,128,1,fp8,fp8,0,7.238090515136719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,4,128,1,float16,fp8,0,10.254517237345377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,8,128,1,float16,float16,0,10.665130615234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,8,128,1,float16,fp8,0,10.477466583251953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,1,128,1,float16,float16,0,5.243631998697917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,64,8,128,1,fp8,fp8,0,7.116832097371419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,1,128,1,float16,fp8,0,5.028490702311198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,64,128,1,float16,fp8,0,4.992368062337239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,64,128,1,float16,float16,0,5.3344160715738935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,64,128,1,fp8,fp8,0,3.7075465520222983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,1,128,1,fp8,fp8,0,3.531514803568522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,2,128,1,float16,fp8,0,5.068655967712402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,2,128,1,fp8,fp8,0,3.518101374308268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,4,128,1,float16,float16,0,5.043381373087565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,4,128,1,float16,fp8,0,5.042640050252278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,4,128,1,fp8,fp8,0,3.534010569254557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,2,128,1,float16,float16,0,5.185530662536621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,8,128,1,float16,float16,0,5.256437301635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,8,128,1,float16,fp8,0,5.080698649088542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,64,8,128,1,fp8,fp8,0,3.60916805267334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,1,128,1,float16,float16,0,2.5898240407307944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,1,128,1,fp8,fp8,0,1.8624587059020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,1,128,1,float16,fp8,0,2.551738739013672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,64,128,1,float16,float16,0,2.627328077952067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,2,128,1,float16,float16,0,2.6349706649780273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,64,128,1,float16,fp8,0,2.571477254231771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,2,128,1,float16,fp8,0,2.5841973622639975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,2,128,1,fp8,fp8,0,1.8604426383972168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,64,128,1,fp8,fp8,0,1.9230720202128093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,4,128,1,float16,float16,0,2.6011786460876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,4,128,1,float16,fp8,0,2.560394605000814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,4,128,1,fp8,fp8,0,1.8670719464619954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,8,128,1,float16,float16,0,2.64139731725057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,8,128,1,float16,fp8,0,2.5551253954569497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,1,128,1,float16,float16,0,1.412245273590088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,1,128,1,float16,fp8,0,1.3863627115885417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,64,8,128,1,fp8,fp8,0,1.9017386436462402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,64,128,1,float16,float16,0,1.405786673227946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,1,128,1,fp8,fp8,0,1.0381333033243816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,64,128,1,float16,fp8,0,1.3864213625590007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,2,128,1,float16,float16,0,1.4080692927042644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,64,128,1,fp8,fp8,0,1.0637599627176921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,4,128,1,float16,float16,0,1.4260586102803547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,2,128,1,fp8,fp8,0,1.0381440321604412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,2,128,1,float16,fp8,0,1.3863892555236816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,4,128,1,float16,fp8,0,1.3867093722025554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,4,128,1,fp8,fp8,0,1.040885368982951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,8,128,1,float16,float16,0,1.4112639427185059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,8,128,1,float16,fp8,0,1.3858826955159504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,64,8,128,1,fp8,fp8,0,1.0478506882985432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,1,128,1,float16,float16,0,21.326858520507812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,1,128,1,float16,fp8,0,20.400624593098957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,2,128,1,float16,float16,0,20.05074691772461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,2,128,1,fp8,fp8,0,14.091045379638672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,2,128,1,float16,fp8,0,20.577067057291668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,1,128,1,fp8,fp8,0,14.27292251586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,4,128,1,float16,float16,0,20.32204818725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,4,128,1,float16,fp8,0,20.12664540608724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,1,128,1,float16,float16,0,10.063791910807291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,4,128,1,fp8,fp8,0,14.853050231933594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,8,128,1,float16,fp8,0,20.616629282633465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,8,128,1,fp8,fp8,0,14.234570821126303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,64,8,128,1,float16,float16,0,20.483285268147785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,64,128,1,float16,float16,0,10.557722727457682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,1,128,1,float16,fp8,0,10.204170862833658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,1,128,1,fp8,fp8,0,7.364634831746419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,64,128,1,fp8,fp8,0,7.50979741414388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,64,128,1,float16,fp8,0,10.463488260904947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,2,128,1,fp8,fp8,0,7.031989415486653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,2,128,1,float16,float16,0,10.2499148050944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,2,128,1,float16,fp8,0,9.955541610717773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,4,128,1,float16,fp8,0,10.087807973225912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,4,128,1,float16,float16,0,10.453930536905924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,4,128,1,fp8,fp8,0,7.4146881103515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,8,128,1,float16,float16,0,10.101792017618815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,1,128,1,float16,fp8,0,4.822592099507649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,1,128,1,float16,float16,0,4.894773483276367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,8,128,1,fp8,fp8,0,7.186506907145183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,64,8,128,1,float16,fp8,0,10.14852778116862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,64,128,1,float16,float16,0,5.19488525390625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,64,128,1,float16,fp8,0,5.06114133199056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,1,128,1,fp8,fp8,0,3.5037972132364907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,2,128,1,float16,float16,0,5.041887919108073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,2,128,1,fp8,fp8,0,3.5107253392537436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,2,128,1,float16,fp8,0,4.815850575764974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,4,128,1,float16,float16,0,5.024885177612305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,4,128,1,fp8,fp8,0,3.525162696838379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,4,128,1,float16,fp8,0,4.8110612233479815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,64,128,1,fp8,fp8,0,3.754495938618978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,8,128,1,float16,float16,0,4.8364213307698565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,1,128,1,float16,float16,0,2.4728213946024575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,1,128,1,float16,fp8,0,2.4348905881245932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,8,128,1,float16,fp8,0,4.930624008178711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,1,128,1,fp8,fp8,0,1.819541295369466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,64,8,128,1,fp8,fp8,0,3.5494187672932944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,64,128,1,float16,float16,0,2.6074719429016113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,2,128,1,float16,float16,0,2.476624011993408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,2,128,1,float16,fp8,0,2.4919625918070474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,64,128,1,float16,fp8,0,2.5264320373535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,2,128,1,fp8,fp8,0,1.8640906016031902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,64,128,1,fp8,fp8,0,1.9489332834879558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,4,128,1,float16,float16,0,2.4781440099080405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,4,128,1,float16,fp8,0,2.4460320472717285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,4,128,1,fp8,fp8,0,1.8330079714457195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,1,128,1,float16,float16,0,1.3152320384979248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,1,128,1,float16,fp8,0,1.3000106811523438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,8,128,1,float16,float16,0,2.4996159871419272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,8,128,1,float16,fp8,0,2.4527360598246255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,64,8,128,1,fp8,fp8,0,1.8432586987813313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,64,128,1,float16,float16,0,1.4553066889444988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,1,128,1,fp8,fp8,0,0.980842669804891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,64,128,1,float16,fp8,0,1.3183519840240479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,64,128,1,fp8,fp8,0,1.0388373533884685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,2,128,1,float16,float16,0,1.3285813331604004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,2,128,1,float16,fp8,0,1.2877493699391682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,2,128,1,fp8,fp8,0,0.984656016031901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,4,128,1,float16,float16,0,1.3186026414235432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,4,128,1,float16,fp8,0,1.3132373491923015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,4,128,1,fp8,fp8,0,0.9864586989084879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,8,128,1,float16,float16,0,1.3191626866658528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,8,128,1,float16,fp8,0,1.2922399838765461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,64,8,128,1,fp8,fp8,0,0.9913067022959391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,1,128,1,float16,float16,0,0.7466346422831217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,1,128,1,float16,fp8,0,0.7196853160858154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,1,128,1,fp8,fp8,0,0.566208004951477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,64,128,1,float16,float16,0,0.7400000095367432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,64,128,1,float16,fp8,0,0.7304213047027588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,2,128,1,float16,float16,0,0.7364532947540283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,64,128,1,fp8,fp8,0,0.5919146537780762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,2,128,1,float16,fp8,0,0.7242133617401123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,2,128,1,fp8,fp8,0,0.5743733247121176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,4,128,1,float16,float16,0,0.7390186786651611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,4,128,1,fp8,fp8,0,0.5695893367131551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,8,128,1,float16,float16,0,0.7428960005442301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,8,128,1,float16,fp8,0,0.7220426400502523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,8,128,1,fp8,fp8,0,0.5721120039621989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,64,4,128,1,float16,fp8,0,0.7220799922943115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,1,128,1,float16,float16,0,12.557428995768229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,1,128,1,float16,fp8,0,12.960676829020182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,1,128,1,fp8,fp8,0,8.82479985555013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,2,128,1,float16,fp8,0,12.272773742675781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,2,128,1,fp8,fp8,0,8.79428799947103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,2,128,1,float16,float16,0,12.588308970133463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,4,128,1,float16,float16,0,12.452821095784506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,4,128,1,float16,fp8,0,12.263322194417318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,1,128,1,float16,float16,0,6.1264909108479815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,4,128,1,fp8,fp8,0,9.000122706095377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,8,128,1,float16,float16,0,13.197354634602865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,8,128,1,float16,fp8,0,12.388309478759766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,64,8,128,1,fp8,fp8,0,9.030511856079102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,64,128,1,float16,float16,0,6.362112045288086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,1,128,1,float16,fp8,0,5.980181376139323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,1,128,1,fp8,fp8,0,4.447237332661946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,64,128,1,float16,fp8,0,6.263071695963542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,64,128,1,fp8,fp8,0,4.780096054077148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,2,128,1,float16,float16,0,6.350181579589844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,2,128,1,float16,fp8,0,5.982245127360026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,2,128,1,fp8,fp8,0,4.44757874806722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,4,128,1,float16,float16,0,6.091503779093425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,4,128,1,fp8,fp8,0,4.445695877075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,4,128,1,float16,fp8,0,5.851749420166016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,8,128,1,float16,float16,0,6.151306788126628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,8,128,1,fp8,fp8,0,4.751167933146159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,64,8,128,1,float16,fp8,0,5.9422346750895185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,1,128,1,float16,float16,0,3.0287094116210938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,1,128,1,fp8,fp8,0,2.2338560422261557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,1,128,1,float16,fp8,0,2.912416140238444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,64,128,1,float16,float16,0,3.2248799006144204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,64,128,1,float16,fp8,0,3.138005256652832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,2,128,1,float16,float16,0,3.069317181905111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,2,128,1,float16,fp8,0,2.921680132548014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,2,128,1,fp8,fp8,0,2.2414933840433755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,64,128,1,fp8,fp8,0,2.4418400128682456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,4,128,1,float16,float16,0,2.988272031148275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,4,128,1,fp8,fp8,0,2.2474239667256675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,4,128,1,float16,fp8,0,2.9216480255126953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,8,128,1,float16,float16,0,3.013141314188639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,8,128,1,float16,fp8,0,3.0360533396402993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,64,8,128,1,fp8,fp8,0,2.2646506627400718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,1,128,1,float16,float16,0,1.547925313313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,1,128,1,float16,fp8,0,1.5121493339538574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,64,128,1,float16,float16,0,1.6469279925028484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,1,128,1,fp8,fp8,0,1.1743040084838867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,2,128,1,float16,float16,0,1.5920693079630535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,64,128,1,float16,fp8,0,1.5832266807556152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,2,128,1,float16,fp8,0,1.5154506365458171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,2,128,1,fp8,fp8,0,1.175872008005778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,64,128,1,fp8,fp8,0,1.2818933327992756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,4,128,1,float16,fp8,0,1.5368480682373047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,4,128,1,fp8,fp8,0,1.1845173041025798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,4,128,1,float16,float16,0,1.5513173739115398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,8,128,1,float16,float16,0,1.6131946245829265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,8,128,1,float16,fp8,0,1.5228533744812012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,1,128,1,float16,float16,0,0.8329599698384603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,1,128,1,float16,fp8,0,0.8131413459777832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,64,8,128,1,fp8,fp8,0,1.1903893152872722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,64,128,1,float16,fp8,0,0.8414239883422852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,64,128,1,float16,float16,0,0.8588693141937256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,1,128,1,fp8,fp8,0,0.6562453508377075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,2,128,1,float16,float16,0,0.834928035736084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,64,128,1,fp8,fp8,0,0.6945919990539551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,2,128,1,float16,fp8,0,0.8149759769439697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,2,128,1,fp8,fp8,0,0.6449973185857137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,4,128,1,float16,fp8,0,0.8175679842631022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,4,128,1,float16,float16,0,0.8365386327107748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,4,128,1,fp8,fp8,0,0.6471146742502848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,8,128,1,float16,float16,0,0.865605354309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,8,128,1,float16,fp8,0,0.8170613447825114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,1,128,1,float16,float16,0,0.4795519908269246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,64,8,128,1,fp8,fp8,0,0.6508906682332357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,64,128,1,float16,float16,0,0.4851093292236328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,64,128,1,float16,fp8,0,0.47570133209228516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,1,128,1,float16,fp8,0,0.46589867273966473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,1,128,1,fp8,fp8,0,0.3715573151906331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,64,128,1,fp8,fp8,0,0.40301334857940674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,2,128,1,float16,float16,0,0.47923731803894043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,2,128,1,float16,fp8,0,0.4702399969100952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,2,128,1,fp8,fp8,0,0.365994652112325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,4,128,1,float16,float16,0,0.481114665667216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,4,128,1,float16,fp8,0,0.469157338142395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,8,128,1,float16,float16,0,0.49110933144887287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,8,128,1,float16,fp8,0,0.4702026844024658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,4,128,1,fp8,fp8,0,0.36708799997965497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,64,8,128,1,fp8,fp8,0,0.36930131912231445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,1,128,1,float16,fp8,0,12.7236696879069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,1,128,1,fp8,fp8,0,9.753472010294596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,2,128,1,float16,float16,0,12.868235270182291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,2,128,1,float16,fp8,0,13.346543629964193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,1,128,1,float16,float16,0,12.805674235026041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,2,128,1,fp8,fp8,0,9.75115712483724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,4,128,1,float16,fp8,0,12.537498474121094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,4,128,1,float16,float16,0,13.01580810546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,1,128,1,float16,float16,0,6.3415679931640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,4,128,1,fp8,fp8,0,9.857861200968424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,8,128,1,float16,float16,0,13.011924743652344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,8,128,1,fp8,fp8,0,10.81658681233724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,1,128,1,float16,fp8,0,6.209130605061849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,64,8,128,1,float16,fp8,0,12.702613830566406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,64,128,1,float16,float16,0,6.790202458699544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,64,128,1,float16,fp8,0,6.603375752766927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,64,128,1,fp8,fp8,0,5.340506871541341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,1,128,1,fp8,fp8,0,4.819642702738444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,2,128,1,float16,fp8,0,6.545418421427409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,2,128,1,float16,float16,0,6.384986877441406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,2,128,1,fp8,fp8,0,4.827333450317383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,4,128,1,float16,float16,0,6.249872207641602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,4,128,1,fp8,fp8,0,4.845317204793294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,4,128,1,float16,fp8,0,6.169616063435872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,8,128,1,float16,float16,0,6.341013590494792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,8,128,1,float16,fp8,0,6.358837127685547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,64,8,128,1,fp8,fp8,0,5.270069440205892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,1,128,1,float16,float16,0,3.0622294743855796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,1,128,1,float16,fp8,0,2.9921493530273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,1,128,1,fp8,fp8,0,2.406991958618164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,64,128,1,float16,fp8,0,3.310271898905436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,64,128,1,float16,float16,0,3.3780959447224936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,2,128,1,float16,float16,0,3.125589370727539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,2,128,1,float16,fp8,0,3.106207847595215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,2,128,1,fp8,fp8,0,2.4102986653645835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,4,128,1,float16,float16,0,3.08463986714681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,4,128,1,float16,fp8,0,2.9943040211995444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,64,128,1,fp8,fp8,0,2.7143732706705728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,4,128,1,fp8,fp8,0,2.420053323109945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,8,128,1,float16,float16,0,3.098181406656901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,8,128,1,float16,fp8,0,3.0494559605916343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,1,128,1,float16,float16,0,1.5738080342610676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,1,128,1,float16,fp8,0,1.531402587890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,1,128,1,fp8,fp8,0,1.2414720058441162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,64,8,128,1,fp8,fp8,0,2.5608533223470054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,2,128,1,float16,float16,0,1.5879146258036296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,64,128,1,float16,float16,0,1.7037760416666667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,2,128,1,float16,fp8,0,1.5371947288513184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,2,128,1,fp8,fp8,0,1.2425333658854167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,64,128,1,float16,fp8,0,1.6728426615397136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,64,128,1,fp8,fp8,0,1.3932852745056152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,4,128,1,float16,float16,0,1.580959955851237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,4,128,1,float16,fp8,0,1.5930347442626953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,4,128,1,fp8,fp8,0,1.2492907047271729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,1,128,1,float16,float16,0,0.8536480267842611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,8,128,1,float16,float16,0,1.5884693463643391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,8,128,1,float16,fp8,0,1.5474026997884114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,64,8,128,1,fp8,fp8,0,1.2592480182647705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,1,128,1,float16,fp8,0,0.8091359933217367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,64,128,1,float16,float16,0,0.8867253462473551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,64,128,1,float16,fp8,0,0.8613119920094808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,1,128,1,fp8,fp8,0,0.6807786623636881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,2,128,1,float16,float16,0,0.8296053409576416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,2,128,1,fp8,fp8,0,0.6622346639633179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,64,128,1,fp8,fp8,0,0.7431466579437256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,2,128,1,float16,fp8,0,0.8088800112406412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,4,128,1,float16,float16,0,0.8356373310089111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,4,128,1,float16,fp8,0,0.8137866655985514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,4,128,1,fp8,fp8,0,0.6663039922714233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,8,128,1,float16,float16,0,0.8736266295115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,8,128,1,fp8,fp8,0,0.6712053616841634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,64,8,128,1,float16,fp8,0,0.8143786589304606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,64,128,1,float16,float16,0,0.477728009223938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,64,128,1,float16,fp8,0,0.4663413365681966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,1,128,1,float16,float16,0,0.4591360092163086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,1,128,1,float16,fp8,0,0.44280532995859784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,64,128,1,fp8,fp8,0,0.40888532002766925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,1,128,1,fp8,fp8,0,0.3816373348236084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,2,128,1,float16,float16,0,0.4589333136876424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,2,128,1,float16,fp8,0,0.44495999813079834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,2,128,1,fp8,fp8,0,0.37371734778086346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,4,128,1,float16,float16,0,0.46015465259552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,4,128,1,float16,fp8,0,0.44860267639160156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,4,128,1,fp8,fp8,0,0.3725866476694743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,8,128,1,float16,fp8,0,0.4618346691131592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,8,128,1,float16,float16,0,0.461733341217041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,64,8,128,1,fp8,fp8,0,0.37532798449198407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,64,128,1,float16,float16,0,0.2795520027478536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,1,128,1,float16,float16,0,0.27347733577092487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,1,128,1,float16,fp8,0,0.2635733286539714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,64,128,1,fp8,fp8,0,0.23812800645828247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,1,128,1,fp8,fp8,0,0.21993066867192587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,2,128,1,float16,float16,0,0.2733813325564067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,2,128,1,float16,fp8,0,0.2646453380584717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,64,128,1,float16,fp8,0,0.27477866411209106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,2,128,1,fp8,fp8,0,0.21521600087483725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,4,128,1,float16,float16,0,0.2753866712252299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,4,128,1,float16,fp8,0,0.26502400636672974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,4,128,1,fp8,fp8,0,0.2164106567700704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,8,128,1,float16,float16,0,0.28221867481867474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,8,128,1,float16,fp8,0,0.26809600989023846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,64,8,128,1,fp8,fp8,0,0.21757866938908896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,1,128,1,float16,fp8,0,7.8979841868082685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,2,128,1,float16,fp8,0,7.875136057535808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,2,128,1,float16,float16,0,8.186213175455729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,2,128,1,fp8,fp8,0,7.143205642700195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,4,128,1,float16,float16,0,8.349023818969727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,4,128,1,float16,fp8,0,7.925445556640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,1,128,1,fp8,fp8,0,6.578762690226237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,1,128,1,float16,float16,0,8.19986661275228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,1,128,1,float16,float16,0,3.990480105082194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,4,128,1,fp8,fp8,0,6.597328186035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,8,128,1,fp8,fp8,0,7.265424092610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,8,128,1,float16,float16,0,8.29867172241211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,64,8,128,1,float16,fp8,0,7.906496047973633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,1,128,1,float16,fp8,0,3.8466132481892905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,64,128,1,float16,fp8,0,4.32315190633138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,64,128,1,float16,float16,0,4.516122817993164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,1,128,1,fp8,fp8,0,3.2326666514078775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,64,128,1,fp8,fp8,0,3.6761385599772134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,2,128,1,fp8,fp8,0,3.2409706115722656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,2,128,1,float16,fp8,0,3.836415926615397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,2,128,1,float16,float16,0,4.355973243713379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,4,128,1,float16,float16,0,4.000346819559733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,4,128,1,float16,fp8,0,3.8586721420288086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,4,128,1,fp8,fp8,0,3.260117212931315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,8,128,1,float16,float16,0,4.067973454793294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,8,128,1,fp8,fp8,0,3.290149370829264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,1,128,1,float16,fp8,0,1.9373599688212078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,1,128,1,float16,float16,0,1.9911306699117024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,64,8,128,1,float16,fp8,0,3.8805227279663086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,1,128,1,fp8,fp8,0,1.6228159268697102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,64,128,1,float16,float16,0,2.2254080772399902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,64,128,1,float16,fp8,0,2.5442934036254883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,2,128,1,float16,float16,0,1.992128054300944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,2,128,1,float16,fp8,0,1.9413760503133137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,64,128,1,fp8,fp8,0,1.8657174110412598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,2,128,1,fp8,fp8,0,1.6290507316589355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,4,128,1,float16,float16,0,2.002474625905355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,4,128,1,float16,fp8,0,1.9383573532104492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,4,128,1,fp8,fp8,0,1.644330660502116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,8,128,1,float16,float16,0,2.135802745819092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,1,128,1,float16,float16,0,1.028213342030843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,1,128,1,float16,fp8,0,0.9970719814300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,8,128,1,float16,fp8,0,1.9496159553527832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,1,128,1,fp8,fp8,0,0.8754666646321615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,64,8,128,1,fp8,fp8,0,1.6533013979593914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,64,128,1,float16,float16,0,1.1380373636881511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,64,128,1,float16,fp8,0,1.1085440317789714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,2,128,1,float16,float16,0,1.031215985616048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,2,128,1,float16,fp8,0,1.0
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,2,128,1,fp8,fp8,0,0.8464693228403727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,4,128,1,float16,float16,0,1.083616018295288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,4,128,1,float16,fp8,0,1.005658706029256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,64,128,1,fp8,fp8,0,0.9692479769388834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,4,128,1,fp8,fp8,0,0.8534826437632242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,8,128,1,float16,float16,0,1.0407040119171143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,1,128,1,float16,float16,0,0.5502826770146688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,8,128,1,float16,fp8,0,1.0090346336364746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,64,8,128,1,fp8,fp8,0,0.8628266652425131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,1,128,1,float16,fp8,0,0.5312319993972778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,64,128,1,float16,float16,0,0.5895359913508097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,64,128,1,float16,fp8,0,0.6677707036336263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,1,128,1,fp8,fp8,0,0.4559626579284668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,2,128,1,float16,float16,0,0.5514560143152872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,64,128,1,fp8,fp8,0,0.5174293518066406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,2,128,1,float16,fp8,0,0.532805323600769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,2,128,1,fp8,fp8,0,0.4591519832611084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,4,128,1,float16,float16,0,0.5518079996109009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,4,128,1,float16,fp8,0,0.5334773461023966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,4,128,1,fp8,fp8,0,0.47496533393859863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,8,128,1,float16,float16,0,0.5546453396479288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,8,128,1,float16,fp8,0,0.5397333304087321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,64,8,128,1,fp8,fp8,0,0.4627679983774821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,64,128,1,float16,float16,0,0.3248053391774495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,64,128,1,float16,fp8,0,0.3176693320274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,1,128,1,float16,float16,0,0.30740267038345337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,1,128,1,float16,fp8,0,0.3067466616630554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,64,128,1,fp8,fp8,0,0.2999733289082845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,1,128,1,fp8,fp8,0,0.2484266757965088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,2,128,1,float16,float16,0,0.31297600269317627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,2,128,1,float16,fp8,0,0.30032533407211304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,2,128,1,fp8,fp8,0,0.2495573361714681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,4,128,1,float16,float16,0,0.31086933612823486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,4,128,1,float16,fp8,0,0.3004853328069051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,4,128,1,fp8,fp8,0,0.25818665822347003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,8,128,1,float16,float16,0,0.31169599294662476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,8,128,1,float16,fp8,0,0.30269867181777954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,64,8,128,1,fp8,fp8,0,0.25310399134953815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,64,128,1,float16,float16,0,0.19498133659362793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,1,128,1,float16,fp8,0,0.17398399114608765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,1,128,1,float16,float16,0,0.1788426637649536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,64,128,1,float16,fp8,0,0.1902773380279541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,1,128,1,fp8,fp8,0,0.15359466274579367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,2,128,1,float16,float16,0,0.1785973310470581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,2,128,1,float16,fp8,0,0.1736533244450887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,64,128,1,fp8,fp8,0,0.16883200407028198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,2,128,1,fp8,fp8,0,0.15282666683197021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,4,128,1,float16,float16,0,0.18183465798695883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,4,128,1,fp8,fp8,0,0.15384533007939658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,4,128,1,float16,fp8,0,0.17193067073822021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,8,128,1,float16,fp8,0,0.17342400550842285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,8,128,1,float16,float16,0,0.1796906590461731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,64,8,128,1,fp8,fp8,0,0.15493866801261902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,1,128,1,float16,fp8,0,7.810133616129558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,1,128,1,fp8,fp8,0,6.382752100626628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,2,128,1,fp8,fp8,0,6.449328104654948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,1,128,1,float16,float16,0,7.628320058186849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,2,128,1,float16,float16,0,7.7600962320963545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,2,128,1,float16,fp8,0,7.646090825398763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,4,128,1,float16,float16,0,7.856442769368489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,4,128,1,float16,fp8,0,7.786928176879883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,1,128,1,float16,float16,0,3.742821375528971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,4,128,1,fp8,fp8,0,6.564085642496745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,8,128,1,float16,fp8,0,8.072565078735352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,8,128,1,float16,float16,0,8.522645314534506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,64,8,128,1,fp8,fp8,0,6.705669403076172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,1,128,1,float16,fp8,0,3.748101234436035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,64,128,1,float16,fp8,0,4.690181414286296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,1,128,1,fp8,fp8,0,3.199647903442383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,64,128,1,fp8,fp8,0,4.000373204549153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,2,128,1,float16,float16,0,3.7500425974527993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,64,128,1,float16,float16,0,4.908826510111491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,2,128,1,float16,fp8,0,4.104330698649089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,2,128,1,fp8,fp8,0,3.227952003479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,4,128,1,float16,float16,0,3.880138715108236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,4,128,1,float16,fp8,0,3.8592958450317383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,4,128,1,fp8,fp8,0,3.2421067555745444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,1,128,1,float16,float16,0,1.875216007232666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,8,128,1,float16,fp8,0,3.9756266276041665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,8,128,1,float16,float16,0,3.957834561665853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,64,8,128,1,fp8,fp8,0,3.3088159561157227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,1,128,1,float16,fp8,0,1.87990935643514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,64,128,1,float16,fp8,0,2.360442638397217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,64,128,1,float16,float16,0,3.184416135152181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,1,128,1,fp8,fp8,0,1.5940213203430176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,64,128,1,fp8,fp8,0,2.0037333170572915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,2,128,1,float16,float16,0,1.8841333389282227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,2,128,1,float16,fp8,0,1.885189374287923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,2,128,1,fp8,fp8,0,1.6027626991271973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,4,128,1,float16,float16,0,1.8961812655131023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,4,128,1,float16,fp8,0,1.9102932612101238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,4,128,1,fp8,fp8,0,1.7107200622558594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,8,128,1,float16,float16,0,1.916543960571289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,8,128,1,fp8,fp8,0,1.6501013437906902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,64,8,128,1,float16,fp8,0,1.9317013422648113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,1,128,1,float16,fp8,0,1.0387519995371501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,1,128,1,float16,float16,0,0.9525279998779297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,64,128,1,float16,float16,0,1.2336533069610596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,1,128,1,fp8,fp8,0,0.7853226661682129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,2,128,1,float16,float16,0,0.9564053217569987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,2,128,1,float16,fp8,0,0.9564639727274576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,2,128,1,fp8,fp8,0,0.8305119673411051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,64,128,1,float16,fp8,0,1.1682666937510173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,64,128,1,fp8,fp8,0,1.0141599973042805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,4,128,1,float16,fp8,0,0.962831974029541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,4,128,1,float16,float16,0,0.9603892962137858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,4,128,1,fp8,fp8,0,0.816805362701416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,8,128,1,float16,float16,0,0.9723573525746664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,1,128,1,float16,float16,0,0.4902720053990682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,8,128,1,float16,fp8,0,0.9714346726735433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,64,8,128,1,fp8,fp8,0,0.8721653620402018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,1,128,1,float16,fp8,0,0.48868266741434735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,64,128,1,float16,float16,0,0.6161280075709025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,1,128,1,fp8,fp8,0,0.4037546714146932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,64,128,1,float16,fp8,0,0.5995466709136963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,64,128,1,fp8,fp8,0,0.5055040121078491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,2,128,1,float16,fp8,0,0.4935786724090576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,2,128,1,fp8,fp8,0,0.4259893496831258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,2,128,1,float16,float16,0,0.4914720058441162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,4,128,1,float16,float16,0,0.49319998423258465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,4,128,1,float16,fp8,0,0.4943893353144328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,4,128,1,fp8,fp8,0,0.4100053310394287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,8,128,1,float16,float16,0,0.5008586645126343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,8,128,1,fp8,fp8,0,0.41494401295979816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,64,8,128,1,float16,fp8,0,0.5008159875869751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,1,128,1,float16,float16,0,0.25707199176152545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,1,128,1,float16,fp8,0,0.25757867097854614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,64,128,1,float16,float16,0,0.4187733332316081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,64,128,1,fp8,fp8,0,0.2620533307393392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,64,128,1,float16,fp8,0,0.31221866607666016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,1,128,1,fp8,fp8,0,0.21357866128285727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,2,128,1,float16,float16,0,0.25853333870569867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,2,128,1,fp8,fp8,0,0.21478400627772012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,2,128,1,float16,fp8,0,0.2593226631482442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,4,128,1,float16,fp8,0,0.2823306719462077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,4,128,1,float16,float16,0,0.2601813276608785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,4,128,1,fp8,fp8,0,0.21506667137145996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,8,128,1,float16,fp8,0,0.2644853393236796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,8,128,1,float16,float16,0,0.2638453245162964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,1,128,1,float16,fp8,0,0.14102933804194132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,64,128,1,float16,float16,0,0.17387733856836954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,1,128,1,float16,float16,0,0.14097066720326742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,64,128,1,float16,fp8,0,0.1690773367881775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,64,128,1,fp8,fp8,0,0.17997332413991293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,1,128,1,fp8,fp8,0,0.10973333319028218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,64,8,128,1,fp8,fp8,0,0.21850132942199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,2,128,1,fp8,fp8,0,0.10991467038790385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,4,128,1,float16,float16,0,0.14317333698272705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,2,128,1,float16,fp8,0,0.142277330160141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,4,128,1,float16,fp8,0,0.14335466424624124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,4,128,1,fp8,fp8,0,0.11733866731325786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,8,128,1,float16,float16,0,0.14541332920392355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,2,128,1,float16,float16,0,0.14221866925557455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,8,128,1,float16,fp8,0,0.1457866628964742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,64,8,128,1,fp8,fp8,0,0.1129813293615977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,1,128,1,float16,float16,0,0.07666666805744171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,1,128,1,float16,fp8,0,0.0753600001335144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,64,128,1,float16,float16,0,0.0953439970811208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,64,128,1,fp8,fp8,0,0.07746666669845581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,64,128,1,float16,fp8,0,0.0918986697991689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,1,128,1,fp8,fp8,0,0.06036800146102905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,2,128,1,fp8,fp8,0,0.06244266529877981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,4,128,1,float16,float16,0,0.07629333436489105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,2,128,1,float16,float16,0,0.07603199779987335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,4,128,1,float16,fp8,0,0.07628266513347626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,4,128,1,fp8,fp8,0,0.062224000692367554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,8,128,1,float16,float16,0,0.07733333110809326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,2,128,1,float16,fp8,0,0.07630399862925212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,8,128,1,float16,fp8,0,0.0772213339805603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,64,8,128,1,fp8,fp8,0,0.06750933329264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,1,128,1,fp8,fp8,0,5.416389465332031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,2,128,1,float16,fp8,0,5.933855692545573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,2,128,1,fp8,fp8,0,5.408213297526042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,1,128,1,float16,fp8,0,5.760527928670247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,4,128,1,float16,fp8,0,6.594320297241211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,1,128,1,float16,float16,0,5.822293599446614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,2,128,1,float16,float16,0,5.956000010172526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,4,128,1,float16,float16,0,6.126773198445638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,1,128,1,float16,float16,0,2.893909454345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,4,128,1,fp8,fp8,0,5.475786844889323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,8,128,1,float16,float16,0,6.231237411499023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,8,128,1,float16,fp8,0,6.2500050862630205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,64,8,128,1,fp8,fp8,0,5.60041618347168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,1,128,1,float16,fp8,0,2.8901761372884116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,64,128,1,float16,float16,0,5.974751790364583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,1,128,1,fp8,fp8,0,2.6738560994466147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,64,128,1,float16,fp8,0,3.991370519002279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,2,128,1,float16,float16,0,2.9037653605143228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,64,128,1,fp8,fp8,0,3.477919896443685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,2,128,1,float16,fp8,0,2.9266347885131836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,2,128,1,fp8,fp8,0,2.6703999837239585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,4,128,1,float16,float16,0,3.053984006245931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,4,128,1,float16,fp8,0,3.308325449625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,4,128,1,fp8,fp8,0,2.7272586822509766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,8,128,1,float16,fp8,0,3.1202398935953775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,1,128,1,float16,float16,0,1.6373119354248047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,8,128,1,float16,float16,0,3.1093438466389975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,1,128,1,float16,fp8,0,1.4577280680338542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,64,8,128,1,fp8,fp8,0,2.783162752787272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,1,128,1,fp8,fp8,0,1.331722656885783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,64,128,1,float16,float16,0,2.0792214075724282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,2,128,1,float16,float16,0,1.6607146263122559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,64,128,1,float16,fp8,0,1.9928053220113118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,2,128,1,fp8,fp8,0,1.3333173592885335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,2,128,1,float16,fp8,0,1.4650400479634602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,4,128,1,float16,float16,0,1.4739573796590169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,64,128,1,fp8,fp8,0,1.7583732604980469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,4,128,1,float16,fp8,0,1.4769760767618816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,4,128,1,fp8,fp8,0,1.354032039642334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,8,128,1,float16,float16,0,1.756597359975179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,8,128,1,fp8,fp8,0,1.3890773455301921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,64,8,128,1,float16,fp8,0,1.5093332926432292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,64,128,1,float16,fp8,0,0.986133337020874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,64,128,1,float16,float16,0,1.0373067061106365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,1,128,1,float16,float16,0,0.7405280272165934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,1,128,1,float16,fp8,0,0.7406400044759115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,1,128,1,fp8,fp8,0,0.6857706705729166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,2,128,1,float16,float16,0,0.7439040342966715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,2,128,1,float16,fp8,0,0.7457546393076578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,64,128,1,fp8,fp8,0,0.881605307261149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,2,128,1,fp8,fp8,0,0.6592479944229126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,4,128,1,float16,fp8,0,0.7486826578776041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,4,128,1,float16,float16,0,0.7474400202433268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,4,128,1,fp8,fp8,0,0.6824533144632975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,8,128,1,float16,float16,0,0.888965368270874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,1,128,1,float16,float16,0,0.3812319835027059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,8,128,1,float16,fp8,0,0.7603840033213297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,64,8,128,1,fp8,fp8,0,0.6973546346028646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,1,128,1,float16,fp8,0,0.3808319965998332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,1,128,1,fp8,fp8,0,0.3552693525950114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,64,128,1,float16,fp8,0,0.4981546799341838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,2,128,1,float16,float16,0,0.3829760154088338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,64,128,1,fp8,fp8,0,0.44050133228302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,64,128,1,float16,float16,0,0.5195253292719523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,2,128,1,float16,fp8,0,0.3828959862391154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,2,128,1,fp8,fp8,0,0.3359413146972656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,4,128,1,float16,float16,0,0.3840426603953044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,4,128,1,float16,fp8,0,0.43252265453338623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,4,128,1,fp8,fp8,0,0.33904532591501874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,8,128,1,float16,float16,0,0.39205865065256756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,8,128,1,float16,fp8,0,0.39076801141103107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,64,8,128,1,fp8,fp8,0,0.3473973274230957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,1,128,1,float16,float16,0,0.21743466456731161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,64,128,1,float16,float16,0,0.2720800042152405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,64,128,1,float16,fp8,0,0.2622239987055461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,1,128,1,float16,fp8,0,0.20044267177581787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,1,128,1,fp8,fp8,0,0.17714667320251465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,64,128,1,fp8,fp8,0,0.22843199968338013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,2,128,1,float16,float16,0,0.20203733444213867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,2,128,1,float16,fp8,0,0.20218666394551596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,2,128,1,fp8,fp8,0,0.18758932749430338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,4,128,1,float16,float16,0,0.20309333006540933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,4,128,1,fp8,fp8,0,0.1789813240369161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,4,128,1,float16,fp8,0,0.2036479910214742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,8,128,1,float16,float16,0,0.2072640061378479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,8,128,1,float16,fp8,0,0.20699199040730795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,64,8,128,1,fp8,fp8,0,0.1813653310139974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,64,128,1,float16,float16,0,0.14752533038457236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,1,128,1,float16,float16,0,0.10993066430091858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,1,128,1,float16,fp8,0,0.10972266395886739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,64,128,1,float16,fp8,0,0.1955946683883667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,64,128,1,fp8,fp8,0,0.12524267037709555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,1,128,1,fp8,fp8,0,0.09168000022570293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,2,128,1,float16,float16,0,0.11070932944615682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,2,128,1,float16,fp8,0,0.11054933071136475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,4,128,1,float16,float16,0,0.11213866869608562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,2,128,1,fp8,fp8,0,0.09215466181437175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,4,128,1,float16,fp8,0,0.11174399654070537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,4,128,1,fp8,fp8,0,0.10015466809272766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,8,128,1,float16,float16,0,0.11408533652623494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,8,128,1,float16,fp8,0,0.11409599582354228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,64,8,128,1,fp8,fp8,0,0.09667199850082397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,64,128,1,float16,float16,0,0.08489599823951721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,1,128,1,float16,float16,0,0.06261866788069408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,1,128,1,float16,fp8,0,0.06372266511122386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,1,128,1,fp8,fp8,0,0.052906667192777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,2,128,1,float16,float16,0,0.06281599899133046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,64,128,1,float16,fp8,0,0.08167466521263123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,2,128,1,float16,fp8,0,0.06345599889755249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,64,128,1,fp8,fp8,0,0.06835733354091644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,2,128,1,fp8,fp8,0,0.052629331747690834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,4,128,1,float16,float16,0,0.06339733302593231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,4,128,1,float16,fp8,0,0.06493333478768666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,4,128,1,fp8,fp8,0,0.05377600093682607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,8,128,1,float16,float16,0,0.06401599943637848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,8,128,1,float16,fp8,0,0.06425599753856659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,64,128,1,float16,float16,0,0.04308266441027323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,64,8,128,1,fp8,fp8,0,0.055904000997543335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,64,128,1,float16,fp8,0,0.04264000058174133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,1,128,1,float16,float16,0,0.03691199918588003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,1,128,1,float16,fp8,0,0.03705599904060364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,1,128,1,fp8,fp8,0,0.03194133440653483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,64,128,1,fp8,fp8,0,0.04033066580692927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,2,128,1,float16,float16,0,0.03698666642109553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,2,128,1,float16,fp8,0,0.0371573343873024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,4,128,1,float16,float16,0,0.037130666275819145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,2,128,1,fp8,fp8,0,0.03166399896144867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,4,128,1,float16,fp8,0,0.03761066744724909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,4,128,1,fp8,fp8,0,0.03370666752258936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,8,128,1,float16,float16,0,0.03746666759252548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,8,128,1,fp8,fp8,0,0.03370666752258936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,64,8,128,1,float16,fp8,0,0.038218667109807335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,1,128,1,float16,float16,0,2.492959976196289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,1,128,1,float16,fp8,0,2.485365390777588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,1,128,1,fp8,fp8,0,2.17740805943807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,2,128,1,float16,float16,0,2.941770553588867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,2,128,1,fp8,fp8,0,2.17575470606486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,2,128,1,float16,fp8,0,2.535930633544922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,4,128,1,float16,float16,0,2.6334826151529946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,4,128,1,float16,fp8,0,2.634335994720459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,4,128,1,fp8,fp8,0,2.2304959297180176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,8,128,1,float16,float16,0,2.681525230407715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,8,128,1,float16,fp8,0,2.6892852783203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,1,128,1,float16,float16,0,1.25438396135966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,64,8,128,1,fp8,fp8,0,2.8015626271565757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,64,128,1,float16,float16,0,1.9263200759887695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,64,128,1,float16,fp8,0,1.835301399230957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,1,128,1,float16,fp8,0,1.256117343902588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,1,128,1,fp8,fp8,0,1.0907680193583171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,2,128,1,float16,float16,0,1.26091734568278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,2,128,1,fp8,fp8,0,1.1437013149261475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,2,128,1,float16,fp8,0,1.2635520299275715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,64,128,1,fp8,fp8,0,1.5393919944763184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,4,128,1,float16,float16,0,1.2676959832509358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,4,128,1,fp8,fp8,0,1.1145280202229817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,4,128,1,float16,fp8,0,1.2718613147735596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,8,128,1,float16,float16,0,1.294858694076538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,8,128,1,float16,fp8,0,1.293824036916097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,64,8,128,1,fp8,fp8,0,1.3808852831522624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,1,128,1,float16,float16,0,0.6388693253199259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,1,128,1,float16,fp8,0,0.6385120153427124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,64,128,1,float16,float16,0,0.9670773347218832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,1,128,1,fp8,fp8,0,0.5333333412806193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,64,128,1,float16,fp8,0,0.9214666684468588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,2,128,1,float16,float16,0,0.6418720086415609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,2,128,1,float16,fp8,0,0.7505119641621908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,2,128,1,fp8,fp8,0,0.5384746789932251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,4,128,1,float16,float16,0,0.6466506719589233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,4,128,1,float16,fp8,0,0.6446666717529297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,64,128,1,fp8,fp8,0,0.7740586598714193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,8,128,1,float16,float16,0,0.6581759850184122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,8,128,1,fp8,fp8,0,0.6768106619517008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,1,128,1,float16,float16,0,0.32797332604726154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,8,128,1,float16,fp8,0,0.6569973230361938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,64,4,128,1,fp8,fp8,0,0.5689013401667277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,1,128,1,float16,fp8,0,0.3272266586621602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,64,128,1,float16,float16,0,0.48394668102264404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,64,128,1,float16,fp8,0,0.46251734097798664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,1,128,1,fp8,fp8,0,0.2964693307876587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,2,128,1,float16,float16,0,0.33077333370844525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,64,128,1,fp8,fp8,0,0.3917226791381836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,2,128,1,float16,fp8,0,0.32866666714350384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,4,128,1,float16,float16,0,0.3315146764119466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,2,128,1,fp8,fp8,0,0.27619733413060504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,4,128,1,float16,fp8,0,0.3309546709060669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,4,128,1,fp8,fp8,0,0.2827413280804952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,8,128,1,float16,float16,0,0.4119679927825928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,8,128,1,float16,fp8,0,0.3373440106709798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,1,128,1,float16,float16,0,0.1734666625658671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,64,8,128,1,fp8,fp8,0,0.29496000210444134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,1,128,1,float16,fp8,0,0.17272533973058066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,64,128,1,float16,float16,0,0.2531200051307678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,64,128,1,float16,fp8,0,0.2417280077934265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,1,128,1,fp8,fp8,0,0.15576533476511636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,2,128,1,float16,float16,0,0.1734293301900228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,2,128,1,float16,fp8,0,0.17378133535385132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,2,128,1,fp8,fp8,0,0.1479573349157969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,4,128,1,float16,float16,0,0.17545600732167563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,64,128,1,fp8,fp8,0,0.20386666059494019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,4,128,1,float16,fp8,0,0.17499200503031412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,4,128,1,fp8,fp8,0,0.14972800016403198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,8,128,1,float16,float16,0,0.21529599030812582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,8,128,1,float16,fp8,0,0.1787253419558207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,64,8,128,1,fp8,fp8,0,0.15587199727694193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,1,128,1,float16,float16,0,0.09556800127029419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,64,128,1,float16,float16,0,0.1376426617304484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,1,128,1,float16,fp8,0,0.09582933783531189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,64,128,1,float16,fp8,0,0.1317813297112783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,1,128,1,fp8,fp8,0,0.08924266695976257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,2,128,1,float16,float16,0,0.09541333715120952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,64,128,1,fp8,fp8,0,0.11785067121187846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,2,128,1,float16,fp8,0,0.09550399581591289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,2,128,1,fp8,fp8,0,0.08383466800053914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,4,128,1,float16,float16,0,0.09602133433024089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,4,128,1,float16,fp8,0,0.09679999947547913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,4,128,1,fp8,fp8,0,0.08404266834259033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,8,128,1,float16,fp8,0,0.09817066788673401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,8,128,1,float16,float16,0,0.11431466539700826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,64,8,128,1,fp8,fp8,0,0.08614933490753174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,64,128,1,float16,float16,0,0.08588266372680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,1,128,1,float16,float16,0,0.056458666920661926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,64,128,1,float16,fp8,0,0.08133333424727122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,1,128,1,float16,fp8,0,0.05653333167235056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,64,128,1,fp8,fp8,0,0.06517866750558217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,2,128,1,float16,fp8,0,0.056186666091283165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,1,128,1,fp8,fp8,0,0.047024001677831016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,2,128,1,float16,float16,0,0.05726933479309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,2,128,1,fp8,fp8,0,0.04735999802748362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,4,128,1,float16,fp8,0,0.057301332553227745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,4,128,1,float16,float16,0,0.05685866872469584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,4,128,1,fp8,fp8,0,0.048207998275756836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,8,128,1,float16,float16,0,0.05764266848564148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,8,128,1,float16,fp8,0,0.057429333527882896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,64,128,1,float16,fp8,0,0.03864533454179764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,64,8,128,1,fp8,fp8,0,0.04929066697756449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,64,128,1,float16,float16,0,0.045237332582473755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,1,128,1,float16,float16,0,0.033285332222779594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,1,128,1,float16,fp8,0,0.03379733363787333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,64,128,1,fp8,fp8,0,0.03824000060558319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,1,128,1,fp8,fp8,0,0.029365333418051403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,2,128,1,float16,float16,0,0.0335359995563825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,2,128,1,float16,fp8,0,0.03405333310365677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,2,128,1,fp8,fp8,0,0.029663999875386555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,4,128,1,float16,float16,0,0.03436266630887985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,4,128,1,float16,fp8,0,0.03450666616360346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,4,128,1,fp8,fp8,0,0.031397332747777305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,8,128,1,float16,float16,0,0.03443733354409536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,8,128,1,float16,fp8,0,0.03470933437347412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,64,128,1,float16,float16,0,0.024959998826185863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,64,128,1,float16,fp8,0,0.02532800038655599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,64,8,128,1,fp8,fp8,0,0.03126399964094162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,64,128,1,fp8,fp8,0,0.023290666441122692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,1,128,1,float16,float16,0,0.022863999009132385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,1,128,1,float16,fp8,0,0.02346666653951009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,1,128,1,fp8,fp8,0,0.021130666136741638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,2,128,1,float16,float16,0,0.023103999594847362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,2,128,1,float16,fp8,0,0.023226665953795116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,2,128,1,fp8,fp8,0,0.0210506667693456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,4,128,1,float16,fp8,0,0.023957334458827972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,4,128,1,float16,float16,0,0.023013333479563396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,4,128,1,fp8,fp8,0,0.021583999196688335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,8,128,1,float16,float16,0,0.023578666150569916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,8,128,1,float16,fp8,0,0.023823998868465424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,64,8,128,1,fp8,fp8,0,0.022042666872342426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,1,128,1,float16,float16,0,1.161904017130534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,1,128,1,float16,fp8,0,1.1602826913197835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,1,128,1,fp8,fp8,0,1.1239039897918701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,2,128,1,float16,fp8,0,1.1656906604766846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,2,128,1,float16,float16,0,1.1640266577402751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,2,128,1,fp8,fp8,0,1.0871893564860027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,4,128,1,float16,fp8,0,1.1707253456115723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,4,128,1,float16,float16,0,1.1703253587086995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,1,128,1,float16,float16,0,0.5918933153152466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,4,128,1,fp8,fp8,0,1.1283946832021077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,8,128,1,float16,fp8,0,1.5520960489908855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,8,128,1,float16,float16,0,1.198800007502238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,64,8,128,1,fp8,fp8,0,1.1816799640655518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,1,128,1,float16,fp8,0,0.591482679049174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,64,128,1,float16,fp8,0,0.9503146807352701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,64,128,1,float16,float16,0,0.9976480007171631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,64,128,1,fp8,fp8,0,0.7750720183054606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,1,128,1,fp8,fp8,0,0.5273066759109497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,2,128,1,float16,float16,0,0.7370453675587972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,2,128,1,float16,fp8,0,0.5954506794611613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,2,128,1,fp8,fp8,0,0.5381653308868408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,4,128,1,float16,float16,0,0.5973653395970663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,4,128,1,float16,fp8,0,0.5979040066401163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,4,128,1,fp8,fp8,0,0.558298667271932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,8,128,1,float16,float16,0,0.6119413375854492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,8,128,1,float16,fp8,0,0.6107999881108602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,64,8,128,1,fp8,fp8,0,0.6869440078735352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,1,128,1,float16,float16,0,0.30558933814366657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,64,128,1,float16,float16,0,0.5053973197937012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,64,128,1,float16,fp8,0,0.4854400157928467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,1,128,1,fp8,fp8,0,0.27350399891535443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,1,128,1,float16,fp8,0,0.30612266063690186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,2,128,1,float16,float16,0,0.3081173300743103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,2,128,1,float16,fp8,0,0.37332268555959064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,2,128,1,fp8,fp8,0,0.2766186594963074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,64,128,1,fp8,fp8,0,0.39074134826660156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,4,128,1,float16,float16,0,0.31038399537404376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,4,128,1,float16,fp8,0,0.31086399157842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,4,128,1,fp8,fp8,0,0.2828320066134135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,8,128,1,float16,fp8,0,0.31703466176986694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,1,128,1,float16,float16,0,0.16353066762288412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,8,128,1,float16,float16,0,0.31758399804433185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,1,128,1,float16,fp8,0,0.16310399770736694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,64,8,128,1,fp8,fp8,0,0.344810684521993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,1,128,1,fp8,fp8,0,0.14617600043614706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,64,128,1,float16,float16,0,0.26495466629664105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,64,128,1,float16,fp8,0,0.2526773413022359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,64,128,1,fp8,fp8,0,0.20220265785853067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,2,128,1,float16,float16,0,0.16381333271662393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,2,128,1,fp8,fp8,0,0.1471946636835734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,2,128,1,float16,fp8,0,0.16380266348520914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,4,128,1,float16,float16,0,0.19714132944742838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,4,128,1,fp8,fp8,0,0.14967999855677286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,4,128,1,float16,fp8,0,0.1653439998626709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,8,128,1,float16,float16,0,0.17051732540130615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,8,128,1,float16,fp8,0,0.1694506605466207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,64,128,1,float16,float16,0,0.14139733711878458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,64,8,128,1,fp8,fp8,0,0.15575466553370157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,64,128,1,fp8,fp8,0,0.1535306672255198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,64,128,1,float16,fp8,0,0.1362559994061788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,1,128,1,float16,float16,0,0.09198400378227234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,1,128,1,float16,fp8,0,0.09247466921806335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,1,128,1,fp8,fp8,0,0.08284266789754231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,2,128,1,float16,float16,0,0.09170132875442505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,2,128,1,float16,fp8,0,0.09222400188446045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,4,128,1,float16,float16,0,0.10477333267529805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,4,128,1,float16,fp8,0,0.09223467111587524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,4,128,1,fp8,fp8,0,0.08372799555460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,2,128,1,fp8,fp8,0,0.08264000217119853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,8,128,1,float16,fp8,0,0.09473066528638203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,8,128,1,float16,float16,0,0.09465066591898601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,64,8,128,1,fp8,fp8,0,0.0849173367023468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,64,128,1,float16,float16,0,0.08103999992211659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,64,128,1,float16,fp8,0,0.10638399918874104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,1,128,1,fp8,fp8,0,0.0469760000705719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,1,128,1,float16,float16,0,0.05251200000445048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,64,128,1,fp8,fp8,0,0.06497066716353099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,1,128,1,float16,fp8,0,0.05310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,2,128,1,float16,fp8,0,0.053173333406448364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,2,128,1,float16,float16,0,0.052757332722345986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,4,128,1,float16,float16,0,0.05422399938106537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,4,128,1,float16,fp8,0,0.053360000252723694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,2,128,1,fp8,fp8,0,0.04683200021584829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,8,128,1,float16,float16,0,0.05406933526198069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,8,128,1,float16,fp8,0,0.05427733560403188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,8,128,1,fp8,fp8,0,0.04900800188382467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,64,4,128,1,fp8,fp8,0,0.04782933493455251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,64,128,1,float16,float16,0,0.03737599899371465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,1,128,1,float16,fp8,0,0.03196800003449122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,64,128,1,fp8,fp8,0,0.03752533346414566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,1,128,1,float16,float16,0,0.03180799881617228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,2,128,1,float16,float16,0,0.03160533308982849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,1,128,1,fp8,fp8,0,0.029418667157491047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,2,128,1,float16,fp8,0,0.03183466692765554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,2,128,1,fp8,fp8,0,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,64,128,1,float16,fp8,0,0.03980266551176707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,4,128,1,float16,float16,0,0.0317546675602595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,8,128,1,float16,fp8,0,0.033013333876927696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,8,128,1,float16,float16,0,0.032399999598662056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,4,128,1,float16,fp8,0,0.03252266595760981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,8,128,1,fp8,fp8,0,0.03161066770553589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,64,4,128,1,fp8,fp8,0,0.03149333347876867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,64,128,1,float16,float16,0,0.023925334215164185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,64,128,1,float16,fp8,0,0.024101334313551586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,64,128,1,fp8,fp8,0,0.02292266736427943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,1,128,1,float16,float16,0,0.021984001000722248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,1,128,1,float16,fp8,0,0.022175999979178112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,2,128,1,float16,float16,0,0.021935999393463135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,2,128,1,float16,fp8,0,0.022266666094462078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,2,128,1,fp8,fp8,0,0.02102400114138921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,1,128,1,fp8,fp8,0,0.021061333517233532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,4,128,1,float16,float16,0,0.02232533444960912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,4,128,1,float16,fp8,0,0.022469334304332733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,4,128,1,fp8,fp8,0,0.022357332209746044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,8,128,1,float16,float16,0,0.02258133391539256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,8,128,1,float16,fp8,0,0.022474666436513264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,64,8,128,1,fp8,fp8,0,0.022085333863894146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,64,128,1,float16,fp8,0,0.019306667149066925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,64,128,1,float16,float16,0,0.01887999971707662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,64,128,1,fp8,fp8,0,0.019402666638294857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,1,128,1,float16,float16,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,1,128,1,fp8,fp8,0,0.018160000443458557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,1,128,1,float16,fp8,0,0.01833600054184596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,2,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,2,128,1,float16,float16,0,0.01811733345190684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,2,128,1,fp8,fp8,0,0.01848000039656957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,4,128,1,float16,fp8,0,0.018405333161354065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,4,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,4,128,1,float16,float16,0,0.01810666670401891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,8,128,1,float16,fp8,0,0.018826667219400406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,8,128,1,float16,float16,0,0.018325333793958027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,64,8,128,1,fp8,fp8,0,0.019194666296243668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,1,128,1,float16,float16,0,0.7230933507283529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,1,128,1,float16,fp8,0,0.7228906949361166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,1,128,1,fp8,fp8,0,0.7796640396118164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,2,128,1,float16,float16,0,0.7266666889190674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,2,128,1,fp8,fp8,0,0.7448586622873942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,2,128,1,float16,fp8,0,0.7258666356404623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,4,128,1,float16,float16,0,0.7320160071055094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,4,128,1,float16,fp8,0,0.7314186890920004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,1,128,1,float16,float16,0,0.3677866856257121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,4,128,1,fp8,fp8,0,0.7689440250396729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,8,128,1,float16,float16,0,0.7445493539174398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,8,128,1,float16,fp8,0,0.9424959818522135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,64,8,128,1,fp8,fp8,0,0.7861066659291586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,1,128,1,float16,fp8,0,0.3682773510615031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,64,128,1,fp8,fp8,0,0.4818933407465617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,64,128,1,float16,float16,0,0.5001813173294067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,1,128,1,fp8,fp8,0,0.3779733180999756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,64,128,1,float16,fp8,0,0.47649065653483075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,2,128,1,float16,float16,0,0.3705173333485921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,2,128,1,float16,fp8,0,0.427621324857076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,2,128,1,fp8,fp8,0,0.38020801544189453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,4,128,1,float16,float16,0,0.3738826513290405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,4,128,1,float16,fp8,0,0.3733919858932495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,4,128,1,fp8,fp8,0,0.38257066408793133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,8,128,1,float16,float16,0,0.3816746473312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,8,128,1,float16,fp8,0,0.37907199064890545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,64,8,128,1,fp8,fp8,0,0.3901760180791219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,1,128,1,float16,fp8,0,0.19210133949915567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,1,128,1,float16,float16,0,0.19090133905410767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,64,128,1,float16,float16,0,0.3887466589609782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,64,128,1,float16,fp8,0,0.2489173412322998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,1,128,1,fp8,fp8,0,0.19989866018295288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,64,128,1,fp8,fp8,0,0.25035732984542847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,2,128,1,float16,float16,0,0.19292799631754556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,2,128,1,float16,fp8,0,0.19328000148137411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,2,128,1,fp8,fp8,0,0.1991573373476664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,4,128,1,float16,float16,0,0.19433599710464478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,4,128,1,float16,fp8,0,0.22779732942581177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,4,128,1,fp8,fp8,0,0.19998933871587118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,8,128,1,float16,float16,0,0.19924267133076987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,8,128,1,fp8,fp8,0,0.2034506599108378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,64,8,128,1,float16,fp8,0,0.19908267259597778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,64,128,1,float16,float16,0,0.13902399937311807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,64,128,1,float16,fp8,0,0.13363200426101685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,64,128,1,fp8,fp8,0,0.13513066371281943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,1,128,1,float16,fp8,0,0.10357866684595744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,1,128,1,float16,float16,0,0.11011733611424764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,1,128,1,fp8,fp8,0,0.11008000373840332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,2,128,1,float16,float16,0,0.10418666402498881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,2,128,1,float16,fp8,0,0.10444266597429912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,2,128,1,fp8,fp8,0,0.11060800155003865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,4,128,1,float16,float16,0,0.10506666700045268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,4,128,1,float16,fp8,0,0.10497066378593445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,4,128,1,fp8,fp8,0,0.1104693313439687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,8,128,1,float16,float16,0,0.12628799676895142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,8,128,1,float16,fp8,0,0.1071573297182719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,64,8,128,1,fp8,fp8,0,0.11117866635322571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,64,128,1,float16,float16,0,0.07860800127188365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,1,128,1,float16,float16,0,0.05845333139101664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,64,128,1,float16,fp8,0,0.07584000130494435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,1,128,1,float16,fp8,0,0.05858133236567179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,1,128,1,fp8,fp8,0,0.0613013356924057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,64,128,1,fp8,fp8,0,0.07743999858697255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,2,128,1,float16,float16,0,0.05843733251094818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,2,128,1,float16,fp8,0,0.059205333391825356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,2,128,1,fp8,fp8,0,0.06142933170000712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,4,128,1,float16,float16,0,0.05909333129723867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,4,128,1,float16,fp8,0,0.05958933134873708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,4,128,1,fp8,fp8,0,0.062234664956728615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,8,128,1,float16,float16,0,0.06195733447869619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,8,128,1,fp8,fp8,0,0.06436799963315327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,64,8,128,1,float16,fp8,0,0.06052800019582113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,64,128,1,float16,fp8,0,0.037632000943024956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,64,128,1,float16,float16,0,0.03898133337497711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,1,128,1,float16,float16,0,0.034517332911491394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,1,128,1,float16,fp8,0,0.034661332766215004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,64,128,1,fp8,fp8,0,0.04370133578777313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,1,128,1,fp8,fp8,0,0.03658133248488108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,2,128,1,float16,float16,0,0.03477333237727483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,2,128,1,float16,fp8,0,0.03462400039037069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,2,128,1,fp8,fp8,0,0.0367999995748202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,4,128,1,float16,float16,0,0.03514666606982549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,4,128,1,float16,fp8,0,0.03548266738653183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,4,128,1,fp8,fp8,0,0.0383146678407987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,8,128,1,float16,float16,0,0.03509333233038584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,8,128,1,float16,fp8,0,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,64,8,128,1,fp8,fp8,0,0.038693333665529885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,64,128,1,float16,fp8,0,0.0249493345618248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,64,128,1,float16,float16,0,0.024911999702453613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,1,128,1,float16,float16,0,0.023002666731675465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,1,128,1,float16,fp8,0,0.023189333577950794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,1,128,1,fp8,fp8,0,0.02438933402299881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,2,128,1,float16,float16,0,0.023402666052182514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,2,128,1,float16,fp8,0,0.023215999205907185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,64,128,1,fp8,fp8,0,0.026575999955336254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,2,128,1,fp8,fp8,0,0.024634666740894318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,4,128,1,float16,float16,0,0.0234400009115537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,4,128,1,float16,fp8,0,0.023589332898457844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,8,128,1,float16,float16,0,0.02385066697994868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,4,128,1,fp8,fp8,0,0.025381334125995636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,8,128,1,float16,fp8,0,0.023813332120577495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,64,8,128,1,fp8,fp8,0,0.025445332129796345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,64,128,1,float16,float16,0,0.018906666586796444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,64,128,1,float16,fp8,0,0.0189280000825723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,1,128,1,float16,float16,0,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,64,128,1,fp8,fp8,0,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,1,128,1,float16,fp8,0,0.017407999684413273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,1,128,1,fp8,fp8,0,0.018207999567190807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,2,128,1,float16,float16,0,0.01770666614174843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,2,128,1,float16,fp8,0,0.017535999417304993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,2,128,1,fp8,fp8,0,0.018613333503405254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,4,128,1,float16,float16,0,0.017749333133300144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,4,128,1,float16,fp8,0,0.018277333428462345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,4,128,1,fp8,fp8,0,0.01945066700379054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,8,128,1,fp8,fp8,0,0.01915733392039935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,8,128,1,float16,fp8,0,0.018144000321626663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,64,8,128,1,float16,float16,0,0.017935999979575474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,64,128,1,float16,fp8,0,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,64,128,1,float16,float16,0,0.016058667252461117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,64,128,1,fp8,fp8,0,0.01735466718673706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,1,128,1,float16,float16,0,0.015781333049138386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,1,128,1,float16,fp8,0,0.016048000504573185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,1,128,1,fp8,fp8,0,0.017136000096797943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,2,128,1,float16,float16,0,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,2,128,1,fp8,fp8,0,0.016970666746298473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,2,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,4,128,1,float16,float16,0,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,4,128,1,float16,fp8,0,0.01597333326935768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,4,128,1,fp8,fp8,0,0.017375999440749485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,8,128,1,float16,float16,0,0.015829333414634068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,8,128,1,fp8,fp8,0,0.017263999829689663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,64,8,128,1,float16,fp8,0,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,1,128,1,float16,float16,0,0.5007146596908569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,1,128,1,fp8,fp8,0,0.5906506776809692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,1,128,1,float16,fp8,0,0.5010293324788412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,2,128,1,float16,fp8,0,0.5047146479288737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,2,128,1,float16,float16,0,0.5043840010960897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,2,128,1,fp8,fp8,0,0.5892533461252848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,4,128,1,float16,float16,0,0.5112266540527344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,4,128,1,float16,fp8,0,0.5684746503829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,1,128,1,float16,float16,0,0.27485867341359455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,4,128,1,fp8,fp8,0,0.5942773421605428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,8,128,1,float16,float16,0,0.5206293265024821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,8,128,1,float16,fp8,0,0.5183786551157633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,64,8,128,1,fp8,fp8,0,0.5984959999720255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,64,128,1,float16,float16,0,0.3201333284378052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,64,128,1,float16,fp8,0,0.3087306618690491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,64,128,1,fp8,fp8,0,0.3561333417892456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,1,128,1,float16,fp8,0,0.27375467618306476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,2,128,1,float16,float16,0,0.2581813335418701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,1,128,1,fp8,fp8,0,0.3059626619021098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,2,128,1,float16,fp8,0,0.2589973409970601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,2,128,1,fp8,fp8,0,0.3065440058708191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,4,128,1,float16,fp8,0,0.26266666253407794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,4,128,1,float16,float16,0,0.2620319922765096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,4,128,1,fp8,fp8,0,0.3081760009129842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,8,128,1,float16,float16,0,0.30435200532277423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,8,128,1,float16,fp8,0,0.2669066588083903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,64,8,128,1,fp8,fp8,0,0.3104853431383769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,64,128,1,float16,float16,0,0.16802666584650675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,1,128,1,float16,float16,0,0.13572266697883606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,1,128,1,float16,fp8,0,0.13540800412495932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,64,128,1,float16,fp8,0,0.16179733475049338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,2,128,1,float16,float16,0,0.13659733533859253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,64,128,1,fp8,fp8,0,0.18563199043273926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,1,128,1,fp8,fp8,0,0.16842132806777954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,2,128,1,float16,fp8,0,0.13683199882507324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,2,128,1,fp8,fp8,0,0.16260266304016113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,4,128,1,float16,float16,0,0.13758933544158936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,4,128,1,float16,fp8,0,0.1377226710319519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,4,128,1,fp8,fp8,0,0.16486933827400208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,8,128,1,fp8,fp8,0,0.17322667439778647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,8,128,1,float16,float16,0,0.1402346690495809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,64,8,128,1,float16,fp8,0,0.14011200269063315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,64,128,1,float16,float16,0,0.09158933162689209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,1,128,1,float16,float16,0,0.07396799822648366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,64,128,1,float16,fp8,0,0.08949866890907288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,64,128,1,fp8,fp8,0,0.10140800476074219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,1,128,1,float16,fp8,0,0.07427733143170674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,1,128,1,fp8,fp8,0,0.08983467022577922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,2,128,1,fp8,fp8,0,0.0881066620349884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,2,128,1,float16,fp8,0,0.07376533250013988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,2,128,1,float16,float16,0,0.07438933352629344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,4,128,1,float16,fp8,0,0.0738560010989507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,4,128,1,fp8,fp8,0,0.0900266667207082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,4,128,1,float16,float16,0,0.07431999842325847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,8,128,1,float16,float16,0,0.07566933333873749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,8,128,1,float16,fp8,0,0.07751466830571492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,1,128,1,float16,float16,0,0.04261866708596548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,64,8,128,1,fp8,fp8,0,0.09150399764378865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,64,128,1,fp8,fp8,0,0.05751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,64,128,1,float16,float16,0,0.04690133531888326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,1,128,1,float16,fp8,0,0.04260266820589701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,1,128,1,fp8,fp8,0,0.050474668542544045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,2,128,1,float16,float16,0,0.0424586683511734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,64,128,1,float16,fp8,0,0.04520533482233683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,2,128,1,float16,fp8,0,0.04222933451334635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,2,128,1,fp8,fp8,0,0.05004799862702688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,4,128,1,float16,float16,0,0.0430026650428772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,4,128,1,fp8,fp8,0,0.051818668842315674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,8,128,1,float16,float16,0,0.04303466777006785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,8,128,1,float16,fp8,0,0.04363200068473816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,8,128,1,fp8,fp8,0,0.052154665191968284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,64,4,128,1,float16,fp8,0,0.04338666796684265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,64,128,1,float16,float16,0,0.02769600103298823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,1,128,1,float16,float16,0,0.026352000733216602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,64,128,1,float16,fp8,0,0.02784000088771184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,1,128,1,float16,fp8,0,0.026922665536403656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,64,128,1,fp8,fp8,0,0.03266666581233343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,1,128,1,fp8,fp8,0,0.03178133318821589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,2,128,1,float16,float16,0,0.026906666656335194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,2,128,1,float16,fp8,0,0.0266239990790685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,2,128,1,fp8,fp8,0,0.03181866556406021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,4,128,1,float16,float16,0,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,4,128,1,float16,fp8,0,0.027114666998386383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,4,128,1,fp8,fp8,0,0.032485333581765495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,8,128,1,float16,float16,0,0.02696000039577484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,8,128,1,fp8,fp8,0,0.03274133304754893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,64,8,128,1,float16,fp8,0,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,64,128,1,float16,fp8,0,0.020053333292404812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,64,128,1,float16,float16,0,0.01989866668979327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,64,128,1,fp8,fp8,0,0.022858666876951855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,1,128,1,float16,fp8,0,0.019120000302791595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,1,128,1,float16,float16,0,0.018645333747069042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,1,128,1,fp8,fp8,0,0.021962667504946392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,2,128,1,float16,float16,0,0.01905599981546402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,2,128,1,float16,fp8,0,0.01921066641807556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,2,128,1,fp8,fp8,0,0.022154666483402252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,4,128,1,float16,fp8,0,0.019173332800467808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,4,128,1,float16,float16,0,0.01926400015751521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,8,128,1,float16,fp8,0,0.019440000255902607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,4,128,1,fp8,fp8,0,0.02271466702222824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,8,128,1,float16,float16,0,0.01932266727089882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,64,8,128,1,fp8,fp8,0,0.022634667654832203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,64,128,1,float16,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,64,128,1,float16,float16,0,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,64,128,1,fp8,fp8,0,0.017903999735911686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,1,128,1,float16,fp8,0,0.01533866673707962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,1,128,1,fp8,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,2,128,1,float16,float16,0,0.015168000012636185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,1,128,1,float16,float16,0,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,2,128,1,float16,fp8,0,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,2,128,1,fp8,fp8,0,0.01770666614174843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,4,128,1,float16,float16,0,0.015306666493415833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,4,128,1,float16,fp8,0,0.015770666301250458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,4,128,1,fp8,fp8,0,0.01791999985774358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,8,128,1,float16,float16,0,0.015706667055686314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,8,128,1,float16,fp8,0,0.01589866727590561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,64,8,128,1,fp8,fp8,0,0.017893332988023758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,64,128,1,float16,float16,0,0.014725333700577417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,64,128,1,float16,fp8,0,0.01523200049996376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,64,128,1,fp8,fp8,0,0.017173333714405697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,1,128,1,float16,float16,0,0.014501333236694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,1,128,1,float16,fp8,0,0.014677333335081736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,1,128,1,fp8,fp8,0,0.016794666647911072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,2,128,1,float16,float16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,4,128,1,float16,float16,0,0.014618666221698126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,2,128,1,fp8,fp8,0,0.016735999534527462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,2,128,1,float16,fp8,0,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,4,128,1,fp8,fp8,0,0.016837333639462788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,4,128,1,float16,fp8,0,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,8,128,1,float16,fp8,0,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,8,128,1,fp8,fp8,0,0.01674666628241539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,64,8,128,1,float16,float16,0,0.01471466695268949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,1,128,1,float16,float16,0,0.4264640013376872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,1,128,1,float16,fp8,0,0.42561598618825275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,2,128,1,float16,float16,0,0.4269333283106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,2,128,1,float16,fp8,0,0.44625600179036456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,2,128,1,fp8,fp8,0,0.5116693178812662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,4,128,1,float16,float16,0,0.4291520118713379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,4,128,1,float16,fp8,0,0.42930134137471515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,1,128,1,fp8,fp8,0,0.5089653333028158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,4,128,1,fp8,fp8,0,0.5125386714935303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,8,128,1,float16,float16,0,0.43345598379770917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,8,128,1,float16,fp8,0,0.43294934431711835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,64,8,128,1,fp8,fp8,0,0.5277920166651408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,1,128,1,float16,float16,0,0.22076799472173056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,64,128,1,float16,float16,0,0.24978133042653403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,64,128,1,float16,fp8,0,0.2536533276240031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,64,128,1,fp8,fp8,0,0.285861333211263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,1,128,1,float16,fp8,0,0.22087999184926352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,1,128,1,fp8,fp8,0,0.26627200841903687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,2,128,1,float16,float16,0,0.2205066680908203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,2,128,1,float16,fp8,0,0.22715733448664346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,4,128,1,float16,float16,0,0.221178670724233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,4,128,1,float16,fp8,0,0.22153600056966147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,2,128,1,fp8,fp8,0,0.2650933265686035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,4,128,1,fp8,fp8,0,0.26556267340977985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,8,128,1,float16,fp8,0,0.22284799814224243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,8,128,1,float16,float16,0,0.2239146629969279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,64,8,128,1,fp8,fp8,0,0.2675199906031291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,64,128,1,float16,float16,0,0.1495253344376882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,1,128,1,float16,fp8,0,0.11597866813341777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,64,128,1,float16,fp8,0,0.12063466509183247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,64,128,1,fp8,fp8,0,0.15229866902033487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,1,128,1,float16,float16,0,0.11672533551851909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,1,128,1,fp8,fp8,0,0.14154666662216187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,2,128,1,float16,float16,0,0.11692800124486287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,2,128,1,float16,fp8,0,0.11675199866294861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,4,128,1,float16,float16,0,0.11577600240707397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,2,128,1,fp8,fp8,0,0.14171733458836874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,4,128,1,float16,fp8,0,0.11654399832089742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,8,128,1,float16,fp8,0,0.11809066931406657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,8,128,1,float16,float16,0,0.11757333079973857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,4,128,1,fp8,fp8,0,0.14289599657058716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,64,8,128,1,fp8,fp8,0,0.14350400368372598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,1,128,1,float16,float16,0,0.06464533507823944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,64,128,1,float16,float16,0,0.06381866832574208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,64,128,1,float16,fp8,0,0.06163200239340464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,64,128,1,fp8,fp8,0,0.09006399909655254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,1,128,1,float16,fp8,0,0.06508799890677135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,2,128,1,float16,fp8,0,0.0645653357108434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,2,128,1,float16,float16,0,0.0647626668214798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,1,128,1,fp8,fp8,0,0.07799466451009114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,2,128,1,fp8,fp8,0,0.07844799757003784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,4,128,1,float16,float16,0,0.06505066653092702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,4,128,1,float16,fp8,0,0.06526400148868561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,8,128,1,float16,float16,0,0.06579733391602834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,4,128,1,fp8,fp8,0,0.07985066870848338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,8,128,1,fp8,fp8,0,0.07994666695594788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,64,8,128,1,float16,fp8,0,0.06588799754778545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,64,128,1,float16,fp8,0,0.03686933219432831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,1,128,1,float16,float16,0,0.03809066613515218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,64,128,1,fp8,fp8,0,0.045498669147491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,64,128,1,float16,float16,0,0.037077332536379494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,1,128,1,float16,fp8,0,0.03824000060558319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,1,128,1,fp8,fp8,0,0.0459146648645401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,2,128,1,float16,float16,0,0.03750933210055033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,2,128,1,float16,fp8,0,0.0383146678407987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,2,128,1,fp8,fp8,0,0.045909335215886436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,4,128,1,float16,float16,0,0.03817066550254822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,4,128,1,float16,fp8,0,0.03833066672086716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,8,128,1,float16,float16,0,0.03851733356714249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,8,128,1,float16,fp8,0,0.03867733230193456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,4,128,1,fp8,fp8,0,0.04659733176231384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,64,8,128,1,fp8,fp8,0,0.046570668617884316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,64,128,1,float16,float16,0,0.024890666206677754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,64,128,1,fp8,fp8,0,0.029279999434947968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,64,128,1,float16,fp8,0,0.02510400116443634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,1,128,1,float16,float16,0,0.024336000283559162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,1,128,1,float16,fp8,0,0.024800000091393787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,1,128,1,fp8,fp8,0,0.029189333319664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,2,128,1,float16,fp8,0,0.02475200096766154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,2,128,1,float16,float16,0,0.024874667326609295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,4,128,1,float16,float16,0,0.02473066747188568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,2,128,1,fp8,fp8,0,0.02926933268706004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,4,128,1,float16,fp8,0,0.02518933266401291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,4,128,1,fp8,fp8,0,0.02942933390537898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,8,128,1,float16,float16,0,0.024453334510326385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,8,128,1,float16,fp8,0,0.02521066615978877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,64,8,128,1,fp8,fp8,0,0.030026666820049286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,64,128,1,float16,fp8,0,0.01851733277241389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,1,128,1,float16,float16,0,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,1,128,1,float16,fp8,0,0.018133333573738735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,64,128,1,fp8,fp8,0,0.021290667355060577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,1,128,1,fp8,fp8,0,0.02111999938885371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,64,128,1,float16,float16,0,0.018085333208243053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,2,128,1,float16,float16,0,0.01791999985774358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,2,128,1,float16,fp8,0,0.018197332819302876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,2,128,1,fp8,fp8,0,0.021312000850836437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,8,128,1,float16,float16,0,0.018250666558742523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,4,128,1,fp8,fp8,0,0.0216799999276797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,4,128,1,float16,fp8,0,0.018357332795858383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,8,128,1,float16,fp8,0,0.018661333868900936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,8,128,1,fp8,fp8,0,0.021162666380405426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,64,4,128,1,float16,float16,0,0.017909333109855652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,64,128,1,float16,float16,0,0.014933332800865173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,64,128,1,float16,fp8,0,0.015237333873907724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,64,128,1,fp8,fp8,0,0.017488000293572743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,1,128,1,float16,float16,0,0.015205333630243937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,1,128,1,float16,fp8,0,0.015541333705186844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,1,128,1,fp8,fp8,0,0.017344000438849132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,2,128,1,float16,float16,0,0.014938666174809137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,2,128,1,fp8,fp8,0,0.01748266691962878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,2,128,1,float16,fp8,0,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,4,128,1,float16,float16,0,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,4,128,1,float16,fp8,0,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,4,128,1,fp8,fp8,0,0.017674667139848072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,8,128,1,float16,float16,0,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,8,128,1,fp8,fp8,0,0.01725333308180173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,64,8,128,1,float16,fp8,0,0.01575999955336253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,64,128,1,float16,float16,0,0.01422400027513504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,1,128,1,float16,float16,0,0.014245333770910898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,64,128,1,fp8,fp8,0,0.01635733370979627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,1,128,1,float16,fp8,0,0.014837333311637243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,64,128,1,float16,fp8,0,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,2,128,1,float16,float16,0,0.0141546664138635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,1,128,1,fp8,fp8,0,0.016506666938463848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,2,128,1,float16,fp8,0,0.014815999815861383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,2,128,1,fp8,fp8,0,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,4,128,1,float16,float16,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,4,128,1,fp8,fp8,0,0.016757333030303318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,4,128,1,float16,fp8,0,0.014597332725922266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,8,128,1,float16,float16,0,0.014474666366974512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,8,128,1,float16,fp8,0,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,64,8,128,1,fp8,fp8,0,0.016607999801635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,1,128,1,float16,float16,0,49.188028971354164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,1,128,1,fp8,fp8,0,31.839007059733074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,2,128,1,float16,fp8,0,50.41016133626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,1,128,1,float16,fp8,0,49.01079305013021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,2,128,1,float16,float16,0,48.9955088297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,2,128,1,fp8,fp8,0,31.887051900227863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,4,128,1,float16,float16,0,49.998799641927086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,4,128,1,float16,fp8,0,48.3923594156901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,1,128,1,float16,float16,0,24.421279907226562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,4,128,1,fp8,fp8,0,31.8773930867513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,8,128,1,float16,fp8,0,50.62776184082031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,48,128,1,fp8,fp8,0,16.179866790771484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,8,128,1,float16,float16,0,49.54314676920573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,1,128,1,float16,fp8,0,24.372703552246094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,48,128,1,float16,fp8,0,24.43774922688802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,48,8,128,1,fp8,fp8,0,31.83635711669922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,48,128,1,float16,float16,0,24.360379536946613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,1,128,1,fp8,fp8,0,15.993002573649088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,2,128,1,float16,float16,0,25.275253295898438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,2,128,1,float16,fp8,0,24.80152638753255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,2,128,1,fp8,fp8,0,15.9935302734375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,4,128,1,float16,float16,0,24.31915791829427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,4,128,1,fp8,fp8,0,15.945087432861328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,4,128,1,float16,fp8,0,24.696683247884113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,8,128,1,float16,float16,0,24.61590830485026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,8,128,1,float16,fp8,0,24.66942850748698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,1,128,1,float16,float16,0,12.448661804199219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,1,128,1,fp8,fp8,0,8.008938471476236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,48,8,128,1,fp8,fp8,0,16.353594462076824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,1,128,1,float16,fp8,0,12.477642059326172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,2,128,1,float16,float16,0,12.503531138102213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,48,128,1,float16,float16,0,12.457045237223307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,2,128,1,float16,fp8,0,12.380293528238932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,2,128,1,fp8,fp8,0,8.069664001464844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,48,128,1,fp8,fp8,0,7.946746826171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,48,128,1,float16,fp8,0,12.362042744954428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,4,128,1,float16,float16,0,12.67624537150065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,4,128,1,fp8,fp8,0,8.03597895304362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,4,128,1,float16,fp8,0,12.500794728597006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,8,128,1,float16,float16,0,12.600629170735678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,8,128,1,float16,fp8,0,12.445322672526041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,1,128,1,float16,float16,0,6.359935760498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,48,8,128,1,fp8,fp8,0,8.004805246988932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,1,128,1,float16,fp8,0,6.4464371999104815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,1,128,1,fp8,fp8,0,4.082645416259766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,48,128,1,float16,float16,0,6.116005579630534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,48,128,1,fp8,fp8,0,4.0732161204020185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,48,128,1,float16,fp8,0,6.2010237375895185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,2,128,1,float16,float16,0,6.412437438964844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,2,128,1,float16,fp8,0,6.3516050974528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,2,128,1,fp8,fp8,0,4.145359992980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,4,128,1,float16,fp8,0,6.2190399169921875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,8,128,1,float16,float16,0,6.369962692260742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,8,128,1,float16,fp8,0,6.155877431233724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,4,128,1,float16,float16,0,6.3941224416097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,8,128,1,fp8,fp8,0,4.109056154886882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,48,4,128,1,fp8,fp8,0,4.099088033040364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,1,128,1,float16,float16,0,28.511263529459637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,1,128,1,float16,fp8,0,28.376678466796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,1,128,1,fp8,fp8,0,18.867029825846355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,2,128,1,float16,fp8,0,28.307520548502605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,2,128,1,float16,float16,0,28.89551035563151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,4,128,1,float16,float16,0,28.478139241536457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,2,128,1,fp8,fp8,0,18.74493916829427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,4,128,1,float16,fp8,0,28.094309488932293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,1,128,1,float16,float16,0,14.164299011230469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,4,128,1,fp8,fp8,0,18.562795003255207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,8,128,1,float16,fp8,0,28.91125742594401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,8,128,1,float16,float16,0,28.526448567708332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,48,8,128,1,fp8,fp8,0,18.58363215128581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,1,128,1,float16,fp8,0,14.070507049560547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,48,128,1,float16,float16,0,14.52658716837565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,1,128,1,fp8,fp8,0,9.37612279256185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,48,128,1,float16,fp8,0,14.106544494628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,2,128,1,float16,float16,0,14.411701202392578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,48,128,1,fp8,fp8,0,9.40831438700358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,2,128,1,float16,fp8,0,14.221871693929037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,2,128,1,fp8,fp8,0,9.356074651082357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,4,128,1,float16,float16,0,14.268112182617188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,4,128,1,fp8,fp8,0,9.502682367960611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,4,128,1,float16,fp8,0,14.476394653320312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,8,128,1,float16,float16,0,14.27090072631836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,1,128,1,float16,float16,0,7.015493392944336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,1,128,1,float16,fp8,0,7.2340749104817705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,8,128,1,float16,fp8,0,14.40762710571289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,48,128,1,float16,float16,0,7.030106862386067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,1,128,1,fp8,fp8,0,4.644426663716634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,48,8,128,1,fp8,fp8,0,9.463130950927734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,48,128,1,float16,fp8,0,6.839237213134766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,2,128,1,float16,float16,0,7.213829040527344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,48,128,1,fp8,fp8,0,4.664165178934733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,2,128,1,float16,fp8,0,7.350666681925456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,2,128,1,fp8,fp8,0,4.729317347208659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,4,128,1,float16,float16,0,7.123776117960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,4,128,1,float16,fp8,0,7.179973602294922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,4,128,1,fp8,fp8,0,4.704506556193034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,8,128,1,float16,float16,0,7.1221974690755205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,8,128,1,float16,fp8,0,7.125328063964844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,48,8,128,1,fp8,fp8,0,4.641743977864583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,1,128,1,float16,float16,0,3.546191851298014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,48,128,1,float16,fp8,0,3.593653361002604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,1,128,1,float16,fp8,0,3.511477470397949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,48,128,1,float16,float16,0,3.6783412297566733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,1,128,1,fp8,fp8,0,2.4582826296488443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,2,128,1,float16,float16,0,3.562101364135742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,2,128,1,float16,fp8,0,3.506223996480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,48,128,1,fp8,fp8,0,2.4467199643452964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,2,128,1,fp8,fp8,0,2.4610719680786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,4,128,1,float16,float16,0,3.6465012232462564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,4,128,1,float16,fp8,0,3.5171521504720054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,4,128,1,fp8,fp8,0,2.485487937927246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,8,128,1,float16,float16,0,3.574655850728353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,8,128,1,float16,fp8,0,3.618533452351888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,48,8,128,1,fp8,fp8,0,2.4678932825724282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,1,128,1,fp8,fp8,0,13.518768310546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,1,128,1,float16,float16,0,20.321365356445312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,1,128,1,float16,fp8,0,19.8690668741862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,2,128,1,float16,fp8,0,20.13421376546224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,2,128,1,fp8,fp8,0,13.210576375325521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,2,128,1,float16,float16,0,20.303243001302082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,4,128,1,float16,float16,0,20.227434794108074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,4,128,1,float16,fp8,0,19.94115702311198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,1,128,1,float16,float16,0,10.05195172627767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,4,128,1,fp8,fp8,0,13.578837076822916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,8,128,1,float16,float16,0,19.945716857910156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,8,128,1,float16,fp8,0,20.172415415445965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,48,8,128,1,fp8,fp8,0,13.618415832519531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,1,128,1,float16,fp8,0,10.04922103881836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,48,128,1,float16,float16,0,10.174277623494467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,48,128,1,fp8,fp8,0,6.752026875813802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,1,128,1,fp8,fp8,0,6.703866958618164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,48,128,1,float16,fp8,0,10.11962636311849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,2,128,1,float16,float16,0,10.118783950805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,2,128,1,float16,fp8,0,10.18285878499349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,2,128,1,fp8,fp8,0,6.669658660888672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,4,128,1,float16,float16,0,10.093770980834961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,4,128,1,float16,fp8,0,10.095130920410156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,4,128,1,fp8,fp8,0,6.633951822916667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,8,128,1,float16,float16,0,10.334469477335611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,8,128,1,float16,fp8,0,10.135610580444336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,1,128,1,float16,fp8,0,4.803621292114258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,1,128,1,float16,float16,0,4.969114621480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,48,8,128,1,fp8,fp8,0,6.721909205118815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,1,128,1,fp8,fp8,0,3.3809280395507812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,48,128,1,float16,float16,0,4.962474822998047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,48,128,1,float16,fp8,0,4.981930732727051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,48,128,1,fp8,fp8,0,3.350250562032064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,2,128,1,float16,float16,0,4.996346791585286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,2,128,1,float16,fp8,0,5.054256121317546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,2,128,1,fp8,fp8,0,3.3404318491617837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,4,128,1,float16,float16,0,4.933274586995442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,4,128,1,float16,fp8,0,5.112623850504558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,4,128,1,fp8,fp8,0,3.3396641413370767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,8,128,1,float16,float16,0,4.997717221577962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,8,128,1,float16,fp8,0,4.909637451171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,48,8,128,1,fp8,fp8,0,3.3819894790649414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,1,128,1,float16,float16,0,2.57914670308431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,1,128,1,float16,fp8,0,2.5327040354410806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,48,128,1,float16,float16,0,2.5468427340189614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,1,128,1,fp8,fp8,0,1.809834639231364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,2,128,1,float16,float16,0,2.563589255015055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,48,128,1,float16,fp8,0,2.54528538386027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,48,128,1,fp8,fp8,0,1.8061013221740723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,2,128,1,float16,fp8,0,2.536362648010254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,2,128,1,fp8,fp8,0,1.8160799344380696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,4,128,1,float16,float16,0,2.5677599906921387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,4,128,1,float16,fp8,0,2.5473920504252114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,4,128,1,fp8,fp8,0,1.8149973551432292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,8,128,1,float16,float16,0,2.5677332878112793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,8,128,1,float16,fp8,0,2.5541812578837075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,48,8,128,1,fp8,fp8,0,1.8134346008300781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,1,128,1,float16,float16,0,26.717147827148438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,1,128,1,float16,fp8,0,27.22021230061849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,1,128,1,fp8,fp8,0,17.454991658528645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,2,128,1,float16,float16,0,26.677871704101562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,2,128,1,fp8,fp8,0,17.50828806559245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,2,128,1,float16,fp8,0,26.9268315633138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,4,128,1,float16,float16,0,26.672869364420574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,4,128,1,float16,fp8,0,26.290372212727863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,1,128,1,float16,float16,0,13.24460220336914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,4,128,1,fp8,fp8,0,17.801930745442707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,8,128,1,fp8,fp8,0,17.758682250976562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,8,128,1,float16,float16,0,27.546404520670574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,48,8,128,1,float16,fp8,0,26.53803253173828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,1,128,1,float16,fp8,0,13.262213389078775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,48,128,1,float16,fp8,0,13.548831939697266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,48,128,1,fp8,fp8,0,9.100159962972006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,48,128,1,float16,float16,0,13.239962259928385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,2,128,1,fp8,fp8,0,8.712170918782553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,2,128,1,float16,float16,0,13.406244913736979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,1,128,1,fp8,fp8,0,8.86130142211914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,2,128,1,float16,fp8,0,13.589781443277994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,4,128,1,float16,float16,0,13.449055989583334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,4,128,1,float16,fp8,0,13.635381062825521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,1,128,1,float16,float16,0,6.670778910319011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,4,128,1,fp8,fp8,0,8.938549041748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,8,128,1,float16,float16,0,13.220991770426432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,8,128,1,fp8,fp8,0,8.920741399129232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,48,8,128,1,float16,fp8,0,13.365370432535807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,48,128,1,float16,fp8,0,6.773759841918945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,48,128,1,float16,float16,0,6.588869094848633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,1,128,1,float16,fp8,0,6.749941507975261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,48,128,1,fp8,fp8,0,4.636938730875651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,1,128,1,fp8,fp8,0,4.328415870666504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,2,128,1,float16,float16,0,6.573317209879558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,2,128,1,fp8,fp8,0,4.382026672363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,2,128,1,float16,fp8,0,6.486768086751302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,4,128,1,float16,fp8,0,6.53770128885905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,4,128,1,float16,float16,0,6.751658757527669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,4,128,1,fp8,fp8,0,4.492757479349772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,8,128,1,float16,float16,0,6.595557530721028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,1,128,1,float16,float16,0,3.275541305541992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,1,128,1,float16,fp8,0,3.2571681340535483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,8,128,1,fp8,fp8,0,4.370944023132324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,48,8,128,1,float16,fp8,0,6.604901631673177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,1,128,1,fp8,fp8,0,2.2684267361958823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,48,128,1,float16,fp8,0,3.270832061767578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,48,128,1,float16,float16,0,3.2468318939208984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,2,128,1,float16,float16,0,3.282911936442057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,48,128,1,fp8,fp8,0,2.3091306686401367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,2,128,1,float16,fp8,0,3.3220640818277993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,2,128,1,fp8,fp8,0,2.268671989440918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,4,128,1,float16,float16,0,3.325999895731608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,4,128,1,fp8,fp8,0,2.2754666010538735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,4,128,1,float16,fp8,0,3.340714772542318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,8,128,1,float16,float16,0,3.2549972534179688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,8,128,1,float16,fp8,0,3.2602666219075522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,1,128,1,float16,float16,0,1.745541254679362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,48,8,128,1,fp8,fp8,0,2.285775979359945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,1,128,1,float16,fp8,0,1.7176480293273926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,48,128,1,float16,float16,0,1.7545653978983562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,1,128,1,fp8,fp8,0,1.2523146470387776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,48,128,1,float16,fp8,0,1.6981493631998699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,2,128,1,float16,float16,0,1.756117343902588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,48,128,1,fp8,fp8,0,1.262122631072998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,2,128,1,float16,fp8,0,1.7221867243448894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,2,128,1,fp8,fp8,0,1.2499199708302815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,4,128,1,float16,float16,0,1.7474452654520671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,4,128,1,float16,fp8,0,1.7465492884318035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,4,128,1,fp8,fp8,0,1.2528479894002278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,8,128,1,float16,float16,0,1.7433172861735027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,8,128,1,float16,fp8,0,1.735365390777588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,48,8,128,1,fp8,fp8,0,1.257424036661784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,1,128,1,float16,float16,0,15.76922607421875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,1,128,1,fp8,fp8,0,10.764405568440756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,1,128,1,float16,fp8,0,15.71883773803711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,2,128,1,float16,float16,0,16.082501729329426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,2,128,1,float16,fp8,0,15.516986846923828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,2,128,1,fp8,fp8,0,10.515087763468424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,4,128,1,float16,float16,0,16.198917388916016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,4,128,1,float16,fp8,0,15.604362487792969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,1,128,1,float16,float16,0,7.852325439453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,4,128,1,fp8,fp8,0,10.64458147684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,8,128,1,float16,float16,0,15.853621164957682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,8,128,1,fp8,fp8,0,10.920282999674479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,48,8,128,1,float16,fp8,0,15.565877278645834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,1,128,1,float16,fp8,0,7.685605367024739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,1,128,1,fp8,fp8,0,5.249989191691081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,48,128,1,fp8,fp8,0,5.83296012878418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,48,128,1,float16,fp8,0,7.707824071248372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,2,128,1,float16,float16,0,7.7248109181722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,48,128,1,float16,float16,0,7.907786687215169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,2,128,1,float16,fp8,0,7.9802506764729815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,2,128,1,fp8,fp8,0,5.254037221272786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,4,128,1,float16,float16,0,7.839413324991862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,4,128,1,float16,fp8,0,7.799258550008138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,4,128,1,fp8,fp8,0,5.495248158772786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,8,128,1,float16,float16,0,7.7905012766520185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,1,128,1,float16,fp8,0,3.8757333755493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,8,128,1,fp8,fp8,0,5.237781206766765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,1,128,1,float16,float16,0,3.900714556376139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,48,8,128,1,float16,fp8,0,7.634725570678711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,1,128,1,fp8,fp8,0,2.6573012669881186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,48,128,1,float16,float16,0,3.9532480239868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,48,128,1,float16,fp8,0,3.7738612492879233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,48,128,1,fp8,fp8,0,2.769909222920736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,2,128,1,float16,float16,0,3.930058797200521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,2,128,1,float16,fp8,0,3.7275412877400718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,2,128,1,fp8,fp8,0,2.654282728830973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,4,128,1,float16,float16,0,3.862074534098307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,4,128,1,float16,fp8,0,3.726773262023926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,4,128,1,fp8,fp8,0,2.678442637125651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,8,128,1,float16,float16,0,3.7730719248453775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,1,128,1,float16,float16,0,1.9639466603597004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,8,128,1,float16,fp8,0,3.77344544728597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,1,128,1,float16,fp8,0,1.9617013931274414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,48,8,128,1,fp8,fp8,0,2.7328640619913735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,48,128,1,float16,fp8,0,1.962106704711914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,1,128,1,fp8,fp8,0,1.4075627326965332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,48,128,1,float16,float16,0,1.971135934193929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,48,128,1,fp8,fp8,0,1.4574826558430989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,2,128,1,float16,float16,0,1.9674186706542969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,2,128,1,float16,fp8,0,1.9416267077128093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,2,128,1,fp8,fp8,0,1.415130615234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,4,128,1,float16,float16,0,2.0012639363606772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,4,128,1,fp8,fp8,0,1.4147413571675618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,4,128,1,float16,fp8,0,1.9422987302144368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,8,128,1,float16,fp8,0,1.947322686513265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,8,128,1,fp8,fp8,0,1.4245279630025227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,48,8,128,1,float16,float16,0,1.9719680150349934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,48,128,1,float16,float16,0,1.0757546424865723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,1,128,1,float16,float16,0,1.075055996576945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,1,128,1,float16,fp8,0,1.0591306686401367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,48,128,1,float16,fp8,0,1.0742613474527996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,48,128,1,fp8,fp8,0,0.8290932973225912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,1,128,1,fp8,fp8,0,0.7938986619313558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,2,128,1,float16,float16,0,1.078501303990682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,2,128,1,fp8,fp8,0,0.7940159638722738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,4,128,1,float16,fp8,0,1.0606719652811687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,4,128,1,float16,float16,0,1.0789493719736736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,2,128,1,float16,fp8,0,1.0573279857635498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,4,128,1,fp8,fp8,0,0.7988693714141846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,8,128,1,float16,float16,0,1.095077355702718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,8,128,1,float16,fp8,0,1.0633066495259602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,48,8,128,1,fp8,fp8,0,0.8019306659698486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,1,128,1,float16,fp8,0,15.177434285481771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,1,128,1,fp8,fp8,0,10.6505495707194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,2,128,1,fp8,fp8,0,11.167322794596354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,2,128,1,float16,fp8,0,15.456432342529297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,1,128,1,float16,float16,0,15.405626932779947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,2,128,1,float16,float16,0,15.233573913574219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,4,128,1,float16,float16,0,15.355669657389322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,4,128,1,float16,fp8,0,15.06537119547526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,1,128,1,float16,float16,0,7.545957565307617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,4,128,1,fp8,fp8,0,10.603200276692709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,8,128,1,float16,float16,0,15.67205810546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,8,128,1,float16,fp8,0,15.985487620035807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,1,128,1,float16,fp8,0,7.434154510498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,48,8,128,1,fp8,fp8,0,11.028303782145182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,48,128,1,float16,fp8,0,7.739909489949544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,48,128,1,float16,float16,0,7.799157460530599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,48,128,1,fp8,fp8,0,5.593722661336263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,1,128,1,fp8,fp8,0,5.2825972239176435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,2,128,1,float16,float16,0,7.730538686116536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,2,128,1,float16,fp8,0,7.887285232543945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,2,128,1,fp8,fp8,0,5.27458127339681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,4,128,1,float16,float16,0,7.833562850952148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,4,128,1,float16,fp8,0,7.4421437581380205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,4,128,1,fp8,fp8,0,5.31057071685791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,8,128,1,float16,float16,0,7.7369810740153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,8,128,1,float16,fp8,0,7.544341405232747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,1,128,1,float16,float16,0,3.637791951497396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,1,128,1,float16,fp8,0,3.6436853408813477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,48,8,128,1,fp8,fp8,0,5.3801015218098955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,48,128,1,float16,float16,0,4.2257280349731445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,1,128,1,fp8,fp8,0,2.6449972788492837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,2,128,1,float16,float16,0,3.6983413696289062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,2,128,1,float16,fp8,0,3.6716906229654946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,48,128,1,float16,fp8,0,3.7480106353759766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,2,128,1,fp8,fp8,0,2.649621327718099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,48,128,1,fp8,fp8,0,2.8553921381632485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,4,128,1,float16,float16,0,3.6503467559814453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,4,128,1,float16,fp8,0,3.7059412002563477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,4,128,1,fp8,fp8,0,2.668538729349772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,8,128,1,float16,float16,0,3.721776008605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,8,128,1,float16,fp8,0,3.6095892588297525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,48,8,128,1,fp8,fp8,0,2.6889654795328775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,1,128,1,float16,float16,0,1.9140106836954753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,48,128,1,float16,float16,0,1.9468320210774739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,1,128,1,float16,fp8,0,1.8434027036031086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,1,128,1,fp8,fp8,0,1.3772320747375488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,2,128,1,float16,fp8,0,1.845568021138509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,48,128,1,float16,fp8,0,1.9387146631876628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,2,128,1,float16,float16,0,1.8820586204528809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,48,128,1,fp8,fp8,0,1.4762132962544758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,2,128,1,fp8,fp8,0,1.4107732772827148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,4,128,1,float16,float16,0,1.8867467244466145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,4,128,1,float16,fp8,0,1.8512639999389648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,4,128,1,fp8,fp8,0,1.386624018351237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,8,128,1,float16,float16,0,1.8906985918680828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,8,128,1,float16,fp8,0,1.883039951324463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,48,8,128,1,fp8,fp8,0,1.4019519488016765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,1,128,1,float16,float16,0,1.0022986729939778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,48,128,1,float16,float16,0,1.017568031946818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,1,128,1,float16,fp8,0,0.9827093283335367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,48,128,1,float16,fp8,0,1.0730186303456624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,1,128,1,fp8,fp8,0,0.7482933203379313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,48,128,1,fp8,fp8,0,0.7900959650675455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,2,128,1,float16,fp8,0,0.9910240173339844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,2,128,1,float16,float16,0,1.0134666760762532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,2,128,1,fp8,fp8,0,0.7504106362660726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,4,128,1,float16,float16,0,1.0048373540242512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,4,128,1,float16,fp8,0,0.9865760008494059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,4,128,1,fp8,fp8,0,0.7682879765828451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,8,128,1,float16,float16,0,1.01145601272583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,8,128,1,float16,fp8,0,0.9898026784261068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,48,8,128,1,fp8,fp8,0,0.7598079840342203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,48,128,1,float16,float16,0,0.5746986468633016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,1,128,1,float16,float16,0,0.5689866542816162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,1,128,1,float16,fp8,0,0.5607093175252279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,48,128,1,float16,fp8,0,0.5607519944508871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,1,128,1,fp8,fp8,0,0.41630399227142334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,48,128,1,fp8,fp8,0,0.4504479964574178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,2,128,1,float16,float16,0,0.5679839849472046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,2,128,1,fp8,fp8,0,0.4170773426691691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,2,128,1,float16,fp8,0,0.5580639839172363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,4,128,1,float16,float16,0,0.5705493291219076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,4,128,1,float16,fp8,0,0.5629813273747762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,4,128,1,fp8,fp8,0,0.4190453290939331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,8,128,1,float16,float16,0,0.5704480012257894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,8,128,1,float16,fp8,0,0.5583306550979614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,48,8,128,1,fp8,fp8,0,0.42191465695699054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,1,128,1,float16,fp8,0,9.151952107747396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,1,128,1,float16,float16,0,9.457157135009766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,2,128,1,float16,float16,0,9.913909276326498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,1,128,1,fp8,fp8,0,6.618762969970703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,2,128,1,float16,fp8,0,9.103520075480143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,2,128,1,fp8,fp8,0,6.624693552652995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,4,128,1,float16,float16,0,9.224224090576172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,4,128,1,float16,fp8,0,9.222037633260092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,1,128,1,float16,float16,0,4.468079884847005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,4,128,1,fp8,fp8,0,6.677114486694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,8,128,1,float16,fp8,0,9.801024119059244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,1,128,1,float16,fp8,0,4.421135902404785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,8,128,1,float16,float16,0,9.529136021931967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,48,8,128,1,fp8,fp8,0,6.71504020690918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,48,128,1,float16,fp8,0,4.646815935770671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,48,128,1,float16,float16,0,4.773903846740723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,48,128,1,fp8,fp8,0,3.5910774866739907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,1,128,1,fp8,fp8,0,3.3301547368367515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,2,128,1,float16,float16,0,4.712090810139974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,2,128,1,float16,fp8,0,4.502229372660319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,2,128,1,fp8,fp8,0,3.33185609181722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,4,128,1,float16,float16,0,4.587562561035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,4,128,1,float16,fp8,0,4.3696746826171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,4,128,1,fp8,fp8,0,3.360293388366699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,8,128,1,float16,float16,0,4.534655888875325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,8,128,1,float16,fp8,0,4.528592109680176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,1,128,1,float16,float16,0,2.2556799252827964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,1,128,1,float16,fp8,0,2.220400015513102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,48,8,128,1,fp8,fp8,0,3.6041653951009116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,48,128,1,float16,float16,0,2.403226693471273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,1,128,1,fp8,fp8,0,1.6874613761901855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,48,128,1,float16,fp8,0,2.3836639722188315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,2,128,1,float16,float16,0,2.2624533971150718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,48,128,1,fp8,fp8,0,1.843008041381836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,2,128,1,float16,fp8,0,2.2103360493977866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,2,128,1,fp8,fp8,0,1.693365255991618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,4,128,1,float16,float16,0,2.341946601867676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,4,128,1,float16,fp8,0,2.2236533164978027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,4,128,1,fp8,fp8,0,1.7004000345865886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,8,128,1,float16,float16,0,2.307941277821859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,8,128,1,float16,fp8,0,2.2376319567362466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,1,128,1,float16,float16,0,1.1772586504618328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,48,8,128,1,fp8,fp8,0,1.7181332906087239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,1,128,1,float16,fp8,0,1.1507573127746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,48,128,1,float16,float16,0,1.223040024439494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,48,128,1,float16,fp8,0,1.2090986569722493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,1,128,1,fp8,fp8,0,0.8900852998097738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,48,128,1,fp8,fp8,0,1.0450186729431152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,2,128,1,float16,float16,0,1.181600014368693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,2,128,1,float16,fp8,0,1.154095967610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,2,128,1,fp8,fp8,0,0.8941760063171387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,4,128,1,float16,float16,0,1.1809600194295247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,4,128,1,float16,fp8,0,1.1550079981486003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,4,128,1,fp8,fp8,0,0.8991359869639078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,8,128,1,float16,float16,0,1.198426644007365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,8,128,1,float16,fp8,0,1.201855977376302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,1,128,1,float16,float16,0,0.637498656908671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,48,128,1,float16,float16,0,0.6558719873428345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,48,8,128,1,fp8,fp8,0,0.9094186623891195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,1,128,1,fp8,fp8,0,0.49167466163635254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,1,128,1,float16,fp8,0,0.6279679934183756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,48,128,1,float16,fp8,0,0.644538680712382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,2,128,1,float16,float16,0,0.6560266812642416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,48,128,1,fp8,fp8,0,0.528277317682902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,2,128,1,float16,fp8,0,0.6238133509953817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,2,128,1,fp8,fp8,0,0.49191999435424805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,4,128,1,float16,fp8,0,0.6287733316421509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,4,128,1,float16,float16,0,0.6432426770528158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,4,128,1,fp8,fp8,0,0.5003093481063843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,8,128,1,float16,float16,0,0.6452159881591797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,8,128,1,float16,fp8,0,0.6452053387959799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,48,8,128,1,fp8,fp8,0,0.4994986852010091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,48,128,1,float16,float16,0,0.3745386600494385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,1,128,1,float16,float16,0,0.37302398681640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,1,128,1,float16,fp8,0,0.3644853432973226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,1,128,1,fp8,fp8,0,0.27932266394297284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,48,128,1,float16,fp8,0,0.36963733037312824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,48,128,1,fp8,fp8,0,0.30406399567921955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,2,128,1,float16,float16,0,0.3795520067214966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,2,128,1,float16,fp8,0,0.36323734124501544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,2,128,1,fp8,fp8,0,0.2788906693458557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,4,128,1,float16,float16,0,0.3749653498331706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,4,128,1,float16,fp8,0,0.365445335706075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,4,128,1,fp8,fp8,0,0.28171199560165405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,8,128,1,float16,float16,0,0.37428800264994305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,8,128,1,float16,fp8,0,0.36606931686401367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,48,8,128,1,fp8,fp8,0,0.2893066604932149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,1,128,1,fp8,fp8,0,7.381338755289714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,1,128,1,float16,fp8,0,9.354901631673178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,1,128,1,float16,float16,0,9.607818603515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,2,128,1,fp8,fp8,0,7.35325304667155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,2,128,1,float16,float16,0,9.6909548441569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,2,128,1,float16,fp8,0,9.382842381795248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,4,128,1,float16,fp8,0,10.105610529581705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,4,128,1,float16,float16,0,9.991717020670572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,1,128,1,float16,float16,0,4.751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,4,128,1,fp8,fp8,0,7.438426971435547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,8,128,1,float16,float16,0,9.713279724121094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,8,128,1,float16,fp8,0,9.732474644978842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,1,128,1,float16,fp8,0,4.672629356384277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,48,8,128,1,fp8,fp8,0,7.444298426310222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,48,128,1,fp8,fp8,0,4.616218566894531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,48,128,1,float16,fp8,0,5.02726395924886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,48,128,1,float16,float16,0,5.157461484273274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,1,128,1,fp8,fp8,0,3.64138126373291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,2,128,1,float16,float16,0,4.738874753316243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,2,128,1,float16,fp8,0,4.540335973103841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,2,128,1,fp8,fp8,0,3.6429227193196616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,4,128,1,float16,float16,0,4.771903991699219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,4,128,1,float16,fp8,0,4.993776003519694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,4,128,1,fp8,fp8,0,3.659909248352051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,8,128,1,float16,fp8,0,4.675941467285156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,8,128,1,float16,float16,0,4.807477315266927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,1,128,1,float16,float16,0,2.4225759506225586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,1,128,1,float16,fp8,0,2.26308806737264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,48,8,128,1,fp8,fp8,0,3.700127919514974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,1,128,1,fp8,fp8,0,1.8236053784688313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,48,128,1,float16,float16,0,2.5484372774759927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,48,128,1,float16,fp8,0,2.52513058980306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,2,128,1,float16,float16,0,2.331914742787679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,2,128,1,float16,fp8,0,2.361834685007731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,48,128,1,fp8,fp8,0,2.0395466486612954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,2,128,1,fp8,fp8,0,1.8237919807434082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,4,128,1,float16,fp8,0,2.2791733741760254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,4,128,1,fp8,fp8,0,1.8316373825073242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,4,128,1,float16,float16,0,2.3352479934692383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,8,128,1,float16,float16,0,2.3758506774902344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,8,128,1,float16,fp8,0,2.291440010070801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,48,8,128,1,fp8,fp8,0,1.9496906598409016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,1,128,1,float16,float16,0,1.198256015777588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,48,128,1,float16,fp8,0,1.2704266707102458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,1,128,1,float16,fp8,0,1.1645653247833252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,48,128,1,float16,float16,0,1.300053358078003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,1,128,1,fp8,fp8,0,0.938922643661499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,2,128,1,float16,float16,0,1.2004000345865886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,2,128,1,float16,fp8,0,1.2074826558430989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,2,128,1,fp8,fp8,0,0.943615992863973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,4,128,1,float16,float16,0,1.2036266326904297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,4,128,1,float16,fp8,0,1.1725653012593586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,4,128,1,fp8,fp8,0,0.9483146667480469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,48,128,1,fp8,fp8,0,1.0610613028208415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,8,128,1,float16,float16,0,1.2187360127766926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,8,128,1,float16,fp8,0,1.1846880118052165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,1,128,1,float16,float16,0,0.6348533233006796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,1,128,1,float16,fp8,0,0.6188799937566122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,48,8,128,1,fp8,fp8,0,0.9965173403422037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,48,128,1,float16,float16,0,0.67194135983785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,1,128,1,fp8,fp8,0,0.503546675046285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,48,128,1,float16,fp8,0,0.6594293514887491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,2,128,1,float16,fp8,0,0.6192373434702555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,2,128,1,float16,float16,0,0.6364906628926595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,48,128,1,fp8,fp8,0,0.563589334487915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,4,128,1,float16,float16,0,0.6635040044784546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,2,128,1,fp8,fp8,0,0.5095626513163248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,4,128,1,float16,fp8,0,0.6220639944076538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,4,128,1,fp8,fp8,0,0.5082559982935587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,8,128,1,float16,float16,0,0.6410239934921265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,8,128,1,float16,fp8,0,0.626362681388855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,48,8,128,1,fp8,fp8,0,0.5140426556269327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,48,128,1,float16,float16,0,0.3667573531468709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,1,128,1,float16,float16,0,0.3540639877319336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,1,128,1,float16,fp8,0,0.3435680071512858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,48,128,1,float16,fp8,0,0.3976373275121053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,48,128,1,fp8,fp8,0,0.3187466661135356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,1,128,1,fp8,fp8,0,0.2750399907430013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,2,128,1,float16,float16,0,0.354912002881368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,2,128,1,float16,fp8,0,0.34486933549245197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,2,128,1,fp8,fp8,0,0.2759360074996948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,4,128,1,float16,float16,0,0.3557013273239136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,4,128,1,float16,fp8,0,0.34612266222635907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,4,128,1,fp8,fp8,0,0.28549333413441974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,8,128,1,float16,float16,0,0.3585280179977417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,8,128,1,float16,fp8,0,0.3476426601409912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,48,8,128,1,fp8,fp8,0,0.28010666370391846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,48,128,1,float16,float16,0,0.2121760050455729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,1,128,1,float16,float16,0,0.19908267259597778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,48,128,1,float16,fp8,0,0.20728532473246256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,48,128,1,fp8,fp8,0,0.1812266707420349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,1,128,1,float16,fp8,0,0.1954773267110189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,1,128,1,fp8,fp8,0,0.16697067022323608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,2,128,1,float16,float16,0,0.19910399119059244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,2,128,1,float16,fp8,0,0.19300266106923422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,2,128,1,fp8,fp8,0,0.16573333740234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,4,128,1,float16,float16,0,0.19986667235692343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,4,128,1,float16,fp8,0,0.19301333030064902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,4,128,1,fp8,fp8,0,0.16607466340065002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,8,128,1,float16,float16,0,0.20478399594624838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,8,128,1,fp8,fp8,0,0.16886399189631143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,48,8,128,1,float16,fp8,0,0.19609065850575766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,1,128,1,float16,float16,0,6.15013313293457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,2,128,1,float16,float16,0,6.160495758056641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,2,128,1,float16,fp8,0,5.993471781412761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,2,128,1,fp8,fp8,0,5.334133148193359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,1,128,1,float16,fp8,0,5.899290720621745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,1,128,1,fp8,fp8,0,4.905872027079265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,4,128,1,float16,float16,0,6.200101216634114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,4,128,1,float16,fp8,0,5.957818349202474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,1,128,1,float16,float16,0,3.018639882405599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,4,128,1,fp8,fp8,0,4.937930742899577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,1,128,1,float16,fp8,0,2.9100319544474282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,8,128,1,float16,float16,0,6.2785600026448565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,8,128,1,fp8,fp8,0,5.487434387207031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,48,8,128,1,float16,fp8,0,6.121338526407878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,48,128,1,fp8,fp8,0,2.767722765604655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,48,128,1,float16,float16,0,3.38154665629069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,1,128,1,fp8,fp8,0,2.4354772567749023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,48,128,1,float16,fp8,0,3.2555627822875977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,2,128,1,float16,fp8,0,2.9112319946289062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,2,128,1,float16,float16,0,3.0231361389160156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,2,128,1,fp8,fp8,0,2.610698699951172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,4,128,1,float16,float16,0,3.059839884440104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,4,128,1,float16,fp8,0,2.9297812779744468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,4,128,1,fp8,fp8,0,2.4542506535847983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,8,128,1,float16,float16,0,3.0663626988728843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,8,128,1,float16,fp8,0,2.9472214380900064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,1,128,1,float16,float16,0,1.513269265492757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,1,128,1,float16,fp8,0,1.464805285135905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,48,8,128,1,fp8,fp8,0,2.4935572942097983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,1,128,1,fp8,fp8,0,1.2277119954427083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,48,128,1,float16,fp8,0,1.9284799893697102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,48,128,1,float16,float16,0,1.6861866315205891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,2,128,1,float16,float16,0,1.5156052907307942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,2,128,1,float16,fp8,0,1.470362663269043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,48,128,1,fp8,fp8,0,1.4089919726053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,2,128,1,fp8,fp8,0,1.233301321665446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,4,128,1,float16,float16,0,1.5192532539367676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,4,128,1,float16,fp8,0,1.4757919311523438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,4,128,1,fp8,fp8,0,1.2932266394297283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,8,128,1,float16,float16,0,1.5377333958943684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,8,128,1,float16,fp8,0,1.4903519948323567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,48,8,128,1,fp8,fp8,0,1.257914702097575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,1,128,1,float16,float16,0,0.785535971323649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,1,128,1,float16,fp8,0,0.7889226277669271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,48,128,1,float16,float16,0,0.8697919845581055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,1,128,1,fp8,fp8,0,0.64847465356191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,2,128,1,float16,float16,0,0.7885759671529134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,48,128,1,fp8,fp8,0,0.7325973510742188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,2,128,1,float16,fp8,0,0.7644373575846354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,2,128,1,fp8,fp8,0,0.6438133319218954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,48,128,1,float16,fp8,0,0.8401652971903483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,4,128,1,float16,float16,0,0.830176035563151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,4,128,1,float16,fp8,0,0.7675039768218994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,4,128,1,fp8,fp8,0,0.6518880128860474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,8,128,1,float16,float16,0,0.7982613245646158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,8,128,1,float16,fp8,0,0.7731786568959554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,1,128,1,float16,float16,0,0.4222346544265747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,48,8,128,1,fp8,fp8,0,0.6586613257726034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,48,128,1,float16,float16,0,0.45370666186014813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,48,128,1,fp8,fp8,0,0.43378134568532306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,1,128,1,fp8,fp8,0,0.3492799997329712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,1,128,1,float16,fp8,0,0.4114346504211426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,2,128,1,float16,float16,0,0.42258667945861816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,48,128,1,float16,fp8,0,0.4426186482111613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,2,128,1,float16,fp8,0,0.41065065066019696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,2,128,1,fp8,fp8,0,0.35096001625061035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,4,128,1,float16,float16,0,0.42422934373219806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,4,128,1,float16,fp8,0,0.4278879960378011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,4,128,1,fp8,fp8,0,0.35282135009765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,8,128,1,float16,fp8,0,0.4153493245442708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,8,128,1,fp8,fp8,0,0.35622398058573407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,1,128,1,float16,float16,0,0.24650667111078897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,48,8,128,1,float16,float16,0,0.4288320144017537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,1,128,1,float16,fp8,0,0.23113600413004556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,48,128,1,float16,float16,0,0.2531306743621826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,48,128,1,float16,fp8,0,0.24724799394607544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,48,128,1,fp8,fp8,0,0.22101332743962607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,1,128,1,fp8,fp8,0,0.1927573283513387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,2,128,1,float16,float16,0,0.2402720053990682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,2,128,1,float16,fp8,0,0.23935999472935995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,4,128,1,float16,float16,0,0.2414133350054423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,4,128,1,float16,fp8,0,0.23481599489847818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,4,128,1,fp8,fp8,0,0.19352533419926962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,2,128,1,fp8,fp8,0,0.19233065843582153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,8,128,1,float16,float16,0,0.24284799893697104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,8,128,1,float16,fp8,0,0.23591999212900797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,48,128,1,float16,float16,0,0.17107200622558594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,48,8,128,1,fp8,fp8,0,0.19581333796183267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,1,128,1,float16,float16,0,0.13847999771436056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,48,128,1,float16,fp8,0,0.14470400412877402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,48,128,1,fp8,fp8,0,0.1316266655921936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,1,128,1,fp8,fp8,0,0.1188320020834605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,1,128,1,float16,fp8,0,0.13459733128547668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,2,128,1,float16,float16,0,0.13823999961217245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,2,128,1,float16,fp8,0,0.13591999808947244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,2,128,1,fp8,fp8,0,0.1188800036907196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,4,128,1,float16,float16,0,0.1386133333047231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,4,128,1,float16,fp8,0,0.13449600338935852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,4,128,1,fp8,fp8,0,0.1202186644077301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,8,128,1,float16,float16,0,0.13970133662223816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,8,128,1,fp8,fp8,0,0.12204800049463908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,48,8,128,1,float16,fp8,0,0.13563199838002524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,1,128,1,float16,float16,0,6.1153920491536455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,1,128,1,fp8,fp8,0,4.853450775146484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,2,128,1,float16,float16,0,5.962330500284831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,2,128,1,float16,fp8,0,5.919247945149739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,2,128,1,fp8,fp8,0,5.02130667368571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,1,128,1,float16,fp8,0,5.843845367431641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,4,128,1,float16,float16,0,6.115914662679036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,4,128,1,float16,fp8,0,6.1600691477457685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,1,128,1,float16,float16,0,2.8793652852376304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,4,128,1,fp8,fp8,0,5.604399998982747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,8,128,1,float16,float16,0,6.196165084838867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,8,128,1,float16,fp8,0,6.205680211385091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,1,128,1,float16,fp8,0,2.851482709248861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,48,128,1,float16,float16,0,3.6940959294637046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,48,128,1,fp8,fp8,0,3.0015945434570312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,1,128,1,fp8,fp8,0,2.4145119984944663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,48,128,1,float16,fp8,0,3.5565013885498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,48,8,128,1,fp8,fp8,0,5.510741551717122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,2,128,1,float16,float16,0,2.958698590596517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,2,128,1,float16,fp8,0,2.8676799138387046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,2,128,1,fp8,fp8,0,2.535050710042318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,4,128,1,float16,float16,0,3.0560105641682944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,4,128,1,fp8,fp8,0,2.8041601181030273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,4,128,1,float16,fp8,0,3.0431092580159507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,8,128,1,float16,float16,0,3.073967933654785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,1,128,1,float16,float16,0,1.550106684366862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,1,128,1,float16,fp8,0,1.436746597290039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,8,128,1,float16,fp8,0,3.1034774780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,48,8,128,1,fp8,fp8,0,2.758869489034017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,48,128,1,float16,float16,0,1.8472107251485188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,48,128,1,float16,fp8,0,1.7851893107096355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,1,128,1,fp8,fp8,0,1.205840031305949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,2,128,1,float16,float16,0,1.5565759340922039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,2,128,1,float16,fp8,0,1.4460585912068684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,48,128,1,fp8,fp8,0,1.5106879870096843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,2,128,1,fp8,fp8,0,1.260426680246989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,4,128,1,float16,float16,0,1.5274240175882976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,4,128,1,float16,fp8,0,1.5077439943949382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,4,128,1,fp8,fp8,0,1.3863520622253418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,8,128,1,float16,float16,0,1.5325706799825032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,8,128,1,float16,fp8,0,1.645909309387207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,1,128,1,float16,float16,0,0.7357920010884603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,48,8,128,1,fp8,fp8,0,1.384042739868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,1,128,1,float16,fp8,0,0.7360266844431559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,48,128,1,float16,float16,0,0.9276373386383057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,48,128,1,float16,fp8,0,0.8971892992655436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,1,128,1,fp8,fp8,0,0.6120800177256266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,48,128,1,fp8,fp8,0,0.7566826343536377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,2,128,1,float16,float16,0,0.7903733253479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,2,128,1,float16,fp8,0,0.7365226745605469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,2,128,1,fp8,fp8,0,0.6324693361918131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,4,128,1,float16,fp8,0,0.7556320031483968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,4,128,1,float16,float16,0,0.7664746443430582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,8,128,1,float16,float16,0,0.7632213433583578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,4,128,1,fp8,fp8,0,0.6962880293528239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,8,128,1,float16,fp8,0,0.7743679682413737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,48,8,128,1,fp8,fp8,0,0.7320693333943685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,48,128,1,float16,float16,0,0.47348801294962567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,1,128,1,float16,fp8,0,0.37909332911173504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,48,128,1,float16,fp8,0,0.45679465929667157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,1,128,1,fp8,fp8,0,0.3165280024210612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,2,128,1,float16,float16,0,0.3829493522644043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,1,128,1,float16,float16,0,0.3821653525034587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,48,128,1,fp8,fp8,0,0.38780800501505536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,2,128,1,float16,fp8,0,0.4111093282699585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,2,128,1,fp8,fp8,0,0.3230560024579366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,4,128,1,float16,float16,0,0.39186131954193115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,4,128,1,float16,fp8,0,0.38761067390441895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,4,128,1,fp8,fp8,0,0.35702399412790936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,8,128,1,float16,float16,0,0.395797332127889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,8,128,1,fp8,fp8,0,0.3609386682510376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,48,128,1,float16,float16,0,0.32180800040562946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,1,128,1,float16,float16,0,0.20306666692097983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,1,128,1,float16,fp8,0,0.2045066754023234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,48,128,1,float16,fp8,0,0.24353599548339844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,1,128,1,fp8,fp8,0,0.1567039986451467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,48,8,128,1,float16,fp8,0,0.3973439931869507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,48,128,1,fp8,fp8,0,0.21007466316223145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,2,128,1,float16,float16,0,0.20430399974187216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,2,128,1,float16,fp8,0,0.21894399325052896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,2,128,1,fp8,fp8,0,0.1573919951915741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,4,128,1,float16,fp8,0,0.2088800072669983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,4,128,1,float16,float16,0,0.2087093393007914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,4,128,1,fp8,fp8,0,0.17467200756072998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,8,128,1,float16,fp8,0,0.2099413275718689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,8,128,1,float16,float16,0,0.21212265888849893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,48,8,128,1,fp8,fp8,0,0.1788960099220276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,1,128,1,float16,float16,0,0.10896000266075134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,48,128,1,float16,float16,0,0.20616533358891806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,1,128,1,float16,fp8,0,0.11012267072995503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,48,128,1,fp8,fp8,0,0.10776000221570332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,48,128,1,float16,fp8,0,0.14271466930707297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,1,128,1,fp8,fp8,0,0.08653333783149719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,2,128,1,float16,fp8,0,0.11054399609565735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,2,128,1,float16,float16,0,0.1104746659596761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,2,128,1,fp8,fp8,0,0.08776533603668213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,4,128,1,float16,float16,0,0.1160640021165212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,4,128,1,float16,fp8,0,0.11205866932868958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,4,128,1,fp8,fp8,0,0.09335466225941975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,8,128,1,float16,float16,0,0.11239467064539592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,8,128,1,float16,fp8,0,0.11212799946467082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,48,8,128,1,fp8,fp8,0,0.09585600097974141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,48,128,1,float16,fp8,0,0.07077866792678833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,48,128,1,float16,float16,0,0.0726986676454544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,48,128,1,fp8,fp8,0,0.0754613329966863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,1,128,1,float16,fp8,0,0.05958933134873708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,1,128,1,float16,float16,0,0.05935466786225637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,2,128,1,float16,float16,0,0.05945600072542826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,1,128,1,fp8,fp8,0,0.048714667558670044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,2,128,1,float16,fp8,0,0.05994666616121928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,4,128,1,float16,float16,0,0.06122133135795593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,4,128,1,float16,fp8,0,0.06211733321348826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,2,128,1,fp8,fp8,0,0.049829334020614624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,4,128,1,fp8,fp8,0,0.05305066704750061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,8,128,1,float16,float16,0,0.06253866851329803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,8,128,1,float16,fp8,0,0.06173333525657654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,48,8,128,1,fp8,fp8,0,0.054058666030565895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,1,128,1,fp8,fp8,0,4.009786605834961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,1,128,1,float16,fp8,0,4.552319844563802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,1,128,1,float16,float16,0,4.672080039978027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,2,128,1,float16,fp8,0,4.5900265375773115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,2,128,1,float16,float16,0,4.7113386789957685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,2,128,1,fp8,fp8,0,4.24567985534668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,4,128,1,float16,float16,0,4.865861256917317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,4,128,1,float16,fp8,0,4.885456085205078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,1,128,1,float16,float16,0,2.28276793162028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,4,128,1,fp8,fp8,0,4.823760032653809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,8,128,1,fp8,fp8,0,4.72218672434489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,8,128,1,float16,float16,0,4.954613367716472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,48,8,128,1,float16,fp8,0,4.974368095397949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,48,128,1,float16,float16,0,3.113178571065267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,1,128,1,float16,fp8,0,2.2284159660339355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,48,128,1,float16,fp8,0,2.971482594807943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,48,128,1,fp8,fp8,0,2.6083200772603354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,2,128,1,float16,fp8,0,2.248026688893636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,2,128,1,float16,float16,0,2.3307040532430015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,2,128,1,fp8,fp8,0,2.1028159459431968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,1,128,1,fp8,fp8,0,2.0130292574564614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,4,128,1,float16,float16,0,2.423978646596273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,4,128,1,float16,fp8,0,2.4283679326375327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,4,128,1,fp8,fp8,0,2.327317396799723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,1,128,1,float16,float16,0,1.1325706640879314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,1,128,1,float16,fp8,0,1.1238346894582112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,8,128,1,fp8,fp8,0,2.3565600713094077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,8,128,1,float16,float16,0,2.4531466166178384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,48,8,128,1,float16,fp8,0,2.484912077585856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,48,128,1,float16,float16,0,1.556015968322754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,1,128,1,fp8,fp8,0,0.9992799758911133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,48,128,1,float16,fp8,0,2.1206347147623696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,2,128,1,float16,float16,0,1.1409813563028972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,48,128,1,fp8,fp8,0,1.3039946556091309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,2,128,1,float16,fp8,0,1.1293013095855713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,2,128,1,fp8,fp8,0,1.051205317179362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,4,128,1,float16,float16,0,1.2107573350270588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,4,128,1,float16,fp8,0,1.2099093596140544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,4,128,1,fp8,fp8,0,1.1606346766153972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,8,128,1,float16,float16,0,1.3541866938273113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,1,128,1,float16,float16,0,0.5762133200963339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,8,128,1,float16,fp8,0,1.2377920150756836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,48,8,128,1,fp8,fp8,0,1.1790400346120198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,1,128,1,float16,fp8,0,0.5739786624908447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,48,128,1,float16,float16,0,0.7828106880187988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,1,128,1,fp8,fp8,0,0.5196266571680704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,48,128,1,float16,fp8,0,0.7427146434783936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,2,128,1,float16,float16,0,0.5814559857050577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,2,128,1,float16,fp8,0,0.5755039850870768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,2,128,1,fp8,fp8,0,0.5283679962158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,4,128,1,float16,float16,0,0.6002080043156942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,48,128,1,fp8,fp8,0,0.6596159934997559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,4,128,1,float16,fp8,0,0.597050666809082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,4,128,1,fp8,fp8,0,0.607312003771464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,8,128,1,float16,fp8,0,0.6122026840845743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,1,128,1,float16,float16,0,0.2972480058670044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,8,128,1,fp8,fp8,0,0.6005386511484782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,48,8,128,1,float16,float16,0,0.6053440173467001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,48,128,1,float16,fp8,0,0.3832106590270996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,48,128,1,float16,float16,0,0.4023679892222087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,1,128,1,float16,fp8,0,0.32500799496968585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,1,128,1,fp8,fp8,0,0.2622026602427165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,2,128,1,float16,float16,0,0.30005866289138794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,48,128,1,fp8,fp8,0,0.3364693323771159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,2,128,1,float16,fp8,0,0.2991786599159241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,4,128,1,float16,float16,0,0.3086506724357605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,4,128,1,float16,fp8,0,0.30713067452112836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,2,128,1,fp8,fp8,0,0.2690560022989909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,4,128,1,fp8,fp8,0,0.30795733133951825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,8,128,1,float16,float16,0,0.31402132908503216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,8,128,1,float16,fp8,0,0.3113066752751668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,48,8,128,1,fp8,fp8,0,0.31007466713587445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,48,128,1,float16,float16,0,0.21146132548650107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,1,128,1,float16,fp8,0,0.17267733812332153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,1,128,1,float16,float16,0,0.15933866302172342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,1,128,1,fp8,fp8,0,0.13167466719945273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,48,128,1,float16,fp8,0,0.2031360069910685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,2,128,1,float16,fp8,0,0.15990400314331055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,2,128,1,float16,float16,0,0.15992533167203268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,48,128,1,fp8,fp8,0,0.18256000677744547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,2,128,1,fp8,fp8,0,0.13247999548912048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,4,128,1,float16,float16,0,0.1798080007235209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,4,128,1,fp8,fp8,0,0.14845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,4,128,1,float16,fp8,0,0.16435733437538147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,8,128,1,fp8,fp8,0,0.15212266643842062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,8,128,1,float16,float16,0,0.16747732957204184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,48,128,1,float16,float16,0,0.12797866264979044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,48,128,1,float16,fp8,0,0.12365333239237468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,1,128,1,float16,float16,0,0.08970666925112407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,48,8,128,1,float16,fp8,0,0.1675306757291158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,48,128,1,fp8,fp8,0,0.12190399567286174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,1,128,1,fp8,fp8,0,0.07230933507283528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,2,128,1,float16,float16,0,0.08945066730181377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,2,128,1,float16,fp8,0,0.08873599767684937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,1,128,1,float16,fp8,0,0.08913066983222961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,2,128,1,fp8,fp8,0,0.0732586681842804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,4,128,1,float16,float16,0,0.09143466750780742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,4,128,1,fp8,fp8,0,0.08262933293978374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,4,128,1,float16,fp8,0,0.09308266639709473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,8,128,1,float16,float16,0,0.091839998960495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,8,128,1,float16,fp8,0,0.09197866916656494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,48,8,128,1,fp8,fp8,0,0.08177599807580312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,48,128,1,float16,float16,0,0.06325866778691609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,48,128,1,float16,fp8,0,0.06118399898211161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,1,128,1,float16,float16,0,0.04980266590913137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,48,128,1,fp8,fp8,0,0.05614933371543884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,2,128,1,float16,fp8,0,0.0503359983364741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,2,128,1,float16,float16,0,0.05018133421738943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,1,128,1,fp8,fp8,0,0.04270400106906891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,1,128,1,float16,fp8,0,0.04985066751639048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,2,128,1,fp8,fp8,0,0.042730664213498436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,4,128,1,float16,float16,0,0.05189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,4,128,1,float16,fp8,0,0.05163733164469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,8,128,1,float16,float16,0,0.05177066723505656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,4,128,1,fp8,fp8,0,0.046911999583244324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,8,128,1,fp8,fp8,0,0.048528000712394714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,48,8,128,1,float16,fp8,0,0.05180799961090088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,48,128,1,float16,fp8,0,0.03465600063403448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,1,128,1,float16,float16,0,0.03179199993610382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,48,128,1,float16,float16,0,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,48,128,1,fp8,fp8,0,0.033402666449546814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,1,128,1,float16,fp8,0,0.03124266614516576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,2,128,1,float16,float16,0,0.03197866678237915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,1,128,1,fp8,fp8,0,0.02886933336655299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,2,128,1,float16,fp8,0,0.0317493329445521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,4,128,1,float16,float16,0,0.03239466746648153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,2,128,1,fp8,fp8,0,0.02921066681543986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,8,128,1,float16,float16,0,0.03270400067170461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,4,128,1,fp8,fp8,0,0.030943999687830608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,8,128,1,float16,fp8,0,0.03259733319282532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,8,128,1,fp8,fp8,0,0.031002665559450786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,48,4,128,1,float16,fp8,0,0.03270400067170461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,1,128,1,float16,float16,0,1.9801920255025227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,1,128,1,float16,fp8,0,2.1745813687642417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,2,128,1,fp8,fp8,0,1.7592533429463704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,2,128,1,float16,float16,0,2.030757268269857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,2,128,1,float16,fp8,0,1.9394079844156902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,4,128,1,float16,float16,0,2.1314667065938315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,1,128,1,fp8,fp8,0,1.672063986460368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,4,128,1,float16,fp8,0,2.1393653551737466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,1,128,1,float16,float16,0,0.9820160071055094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,4,128,1,fp8,fp8,0,2.0162293116251626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,8,128,1,fp8,fp8,0,2.103013356526693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,8,128,1,float16,fp8,0,2.175642649332682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,1,128,1,float16,fp8,0,0.9811786810557047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,48,8,128,1,float16,float16,0,2.4445494016011557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,48,128,1,float16,float16,0,1.4297812779744465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,48,128,1,float16,fp8,0,1.3798559506734211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,48,128,1,fp8,fp8,0,1.156272013982137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,1,128,1,fp8,fp8,0,0.8269920349121094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,2,128,1,float16,float16,0,0.9923893610636393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,2,128,1,float16,fp8,0,0.9821706612904867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,2,128,1,fp8,fp8,0,0.8835146427154541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,4,128,1,float16,fp8,0,1.0490240255991619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,4,128,1,float16,float16,0,1.1532266934712727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,4,128,1,fp8,fp8,0,1.0059146881103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,8,128,1,float16,fp8,0,1.0844053427378337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,8,128,1,float16,float16,0,1.0728586514790852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,1,128,1,float16,float16,0,0.5595093170801798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,1,128,1,float16,fp8,0,0.4979360103607178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,48,8,128,1,fp8,fp8,0,1.0469919840494792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,48,128,1,float16,float16,0,0.7197333176930746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,1,128,1,fp8,fp8,0,0.4233440160751343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,48,128,1,float16,fp8,0,0.6984106699625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,2,128,1,float16,float16,0,0.504810651143392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,2,128,1,fp8,fp8,0,0.4427093267440796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,2,128,1,float16,fp8,0,0.5685760180155436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,48,128,1,fp8,fp8,0,0.5777546564737955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,4,128,1,float16,float16,0,0.5253173510233561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,4,128,1,float16,fp8,0,0.5200586716334025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,4,128,1,fp8,fp8,0,0.5100106795628866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,8,128,1,float16,fp8,0,0.530847986539205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,8,128,1,fp8,fp8,0,0.5284800132115682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,48,128,1,float16,float16,0,0.5698933204015096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,1,128,1,float16,float16,0,0.2587040066719055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,1,128,1,float16,fp8,0,0.25783467292785645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,48,8,128,1,float16,float16,0,0.5319466590881348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,48,128,1,float16,fp8,0,0.3539946476618449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,1,128,1,fp8,fp8,0,0.21894933780034384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,2,128,1,float16,float16,0,0.2597600022951762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,48,128,1,fp8,fp8,0,0.29971200227737427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,2,128,1,fp8,fp8,0,0.22886399428049722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,2,128,1,float16,fp8,0,0.29307732979456586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,4,128,1,float16,float16,0,0.2714346647262573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,4,128,1,fp8,fp8,0,0.2653813362121582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,4,128,1,float16,fp8,0,0.26828799645106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,8,128,1,float16,float16,0,0.2734453280766805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,8,128,1,float16,fp8,0,0.272271990776062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,1,128,1,float16,float16,0,0.13850133617719015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,48,128,1,float16,fp8,0,0.2813386718432109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,1,128,1,float16,fp8,0,0.1400106648604075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,48,128,1,float16,float16,0,0.1979573369026184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,48,8,128,1,fp8,fp8,0,0.2716053326924642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,48,128,1,fp8,fp8,0,0.1670186718304952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,1,128,1,fp8,fp8,0,0.11796800295511882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,2,128,1,float16,float16,0,0.14009599884351095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,2,128,1,float16,fp8,0,0.13966400424639383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,2,128,1,fp8,fp8,0,0.12664000193277994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,4,128,1,float16,fp8,0,0.1441973348458608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,8,128,1,float16,float16,0,0.1474026640256246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,4,128,1,float16,float16,0,0.14467733105023703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,4,128,1,fp8,fp8,0,0.13475733002026877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,8,128,1,fp8,fp8,0,0.14005866646766663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,48,8,128,1,float16,fp8,0,0.14656000336011252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,48,128,1,float16,float16,0,0.11800000071525574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,48,128,1,float16,fp8,0,0.15837867061297098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,1,128,1,float16,float16,0,0.07982933521270752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,48,128,1,fp8,fp8,0,0.0897226631641388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,1,128,1,float16,fp8,0,0.08101866642634074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,2,128,1,float16,float16,0,0.08080000181992848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,2,128,1,float16,fp8,0,0.08091199894746144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,1,128,1,fp8,fp8,0,0.06517866750558217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,2,128,1,fp8,fp8,0,0.06659733255704244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,4,128,1,float16,float16,0,0.08708799878756206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,4,128,1,float16,fp8,0,0.08293333152929942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,4,128,1,fp8,fp8,0,0.07303466896216075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,8,128,1,float16,float16,0,0.08417066931724548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,8,128,1,float16,fp8,0,0.08408533533414204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,48,8,128,1,fp8,fp8,0,0.07520000139872234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,1,128,1,float16,fp8,0,0.045226668318112694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,48,128,1,float16,fp8,0,0.05736533304055532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,1,128,1,float16,float16,0,0.04510400195916494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,48,128,1,fp8,fp8,0,0.06332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,48,128,1,float16,float16,0,0.059749335050582886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,2,128,1,fp8,fp8,0,0.03902400036652883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,2,128,1,float16,fp8,0,0.045514668027559914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,1,128,1,fp8,fp8,0,0.03818133225043615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,2,128,1,float16,float16,0,0.04582933088143667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,4,128,1,fp8,fp8,0,0.042863999803860985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,4,128,1,float16,float16,0,0.04772266745567322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,4,128,1,float16,fp8,0,0.04781333108743032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,8,128,1,float16,fp8,0,0.04866666595141093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,8,128,1,float16,float16,0,0.04756799836953481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,48,8,128,1,fp8,fp8,0,0.04292800029118856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,48,128,1,float16,fp8,0,0.03382933388153712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,1,128,1,float16,float16,0,0.030415999392668407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,48,128,1,fp8,fp8,0,0.03253333270549774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,48,128,1,float16,float16,0,0.03344533344109853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,1,128,1,fp8,fp8,0,0.027050666511058807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,2,128,1,float16,float16,0,0.03035733352104823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,2,128,1,float16,fp8,0,0.0306986669699351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,1,128,1,float16,fp8,0,0.03031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,4,128,1,float16,float16,0,0.03127466638882955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,4,128,1,float16,fp8,0,0.03170666595300039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,2,128,1,fp8,fp8,0,0.027589333554108936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,8,128,1,float16,float16,0,0.031311998764673867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,8,128,1,float16,fp8,0,0.03142933299144109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,4,128,1,fp8,fp8,0,0.029189333319664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,48,128,1,float16,float16,0,0.023717333873112995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,1,128,1,float16,float16,0,0.022426667312781017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,48,128,1,float16,fp8,0,0.02386666586001714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,48,128,1,fp8,fp8,0,0.022783999641736347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,48,8,128,1,fp8,fp8,0,0.028751999139785767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,1,128,1,float16,fp8,0,0.022991999983787537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,1,128,1,fp8,fp8,0,0.020448000480731327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,2,128,1,float16,fp8,0,0.022367998957633972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,4,128,1,float16,float16,0,0.02260799954334895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,4,128,1,float16,fp8,0,0.023520000278949738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,2,128,1,float16,float16,0,0.022074667116006214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,8,128,1,float16,float16,0,0.022522665560245514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,4,128,1,fp8,fp8,0,0.021701333423455555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,2,128,1,fp8,fp8,0,0.02082666630546252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,8,128,1,float16,fp8,0,0.022672000030676525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,48,8,128,1,fp8,fp8,0,0.021573332448800404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,1,128,1,float16,float16,0,0.908128023147583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,1,128,1,float16,fp8,0,1.0633653004964192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,2,128,1,float16,float16,0,0.9345920085906982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,4,128,1,float16,fp8,0,0.9657226403554281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,2,128,1,fp8,fp8,0,0.8783466815948486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,2,128,1,float16,fp8,0,0.9159733454386393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,4,128,1,float16,float16,0,0.9939466317494711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,1,128,1,fp8,fp8,0,0.8394880294799805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,4,128,1,fp8,fp8,0,1.0225120385487874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,1,128,1,float16,float16,0,0.4670720100402832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,8,128,1,float16,float16,0,1.218015988667806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,8,128,1,float16,fp8,0,1.0088906288146973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,48,8,128,1,fp8,fp8,0,1.0445386568705242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,1,128,1,float16,fp8,0,0.46377066771189374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,48,128,1,fp8,fp8,0,0.5813173453013102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,48,128,1,float16,fp8,0,0.7191359996795654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,48,128,1,float16,float16,0,0.7538613478342692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,1,128,1,fp8,fp8,0,0.42136534055074054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,2,128,1,float16,float16,0,0.5525759855906168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,2,128,1,float16,fp8,0,0.4700479904810588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,2,128,1,fp8,fp8,0,0.44120001792907715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,4,128,1,float16,float16,0,0.49876264731089276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,4,128,1,float16,fp8,0,0.48930132389068604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,8,128,1,float16,float16,0,0.5034453471501669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,4,128,1,fp8,fp8,0,0.5140906572341919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,8,128,1,fp8,fp8,0,0.6008479992548624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,48,8,128,1,float16,fp8,0,0.5073866844177246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,1,128,1,float16,float16,0,0.24338134129842123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,48,128,1,float16,float16,0,0.38489067554473877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,48,128,1,float16,fp8,0,0.3696586688359578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,1,128,1,float16,fp8,0,0.2430773377418518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,2,128,1,float16,float16,0,0.28594666719436646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,2,128,1,float16,fp8,0,0.24709866444269815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,1,128,1,fp8,fp8,0,0.2176213264465332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,48,128,1,fp8,fp8,0,0.29576534032821655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,2,128,1,fp8,fp8,0,0.2258560061454773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,4,128,1,float16,float16,0,0.25854400793711346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,4,128,1,float16,fp8,0,0.255349338054657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,4,128,1,fp8,fp8,0,0.2622186740239461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,8,128,1,float16,float16,0,0.32013867298762005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,48,128,1,float16,float16,0,0.2027519941329956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,1,128,1,float16,float16,0,0.13315733273824057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,8,128,1,float16,fp8,0,0.2587946653366089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,48,8,128,1,fp8,fp8,0,0.2704640030860901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,1,128,1,float16,fp8,0,0.13336533308029175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,1,128,1,fp8,fp8,0,0.12487467130025227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,48,128,1,float16,fp8,0,0.1941866676012675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,48,128,1,fp8,fp8,0,0.15756799777348837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,2,128,1,float16,float16,0,0.13540800412495932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,2,128,1,float16,fp8,0,0.1345866620540619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,4,128,1,float16,fp8,0,0.1397813359896342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,2,128,1,fp8,fp8,0,0.11901866396268208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,4,128,1,float16,float16,0,0.14127467075983682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,4,128,1,fp8,fp8,0,0.13909332950909933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,8,128,1,float16,float16,0,0.1437173287073771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,8,128,1,float16,fp8,0,0.14174399773279825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,48,8,128,1,fp8,fp8,0,0.13945600390434265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,48,128,1,float16,float16,0,0.11011200149854024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,1,128,1,float16,fp8,0,0.0790773332118988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,48,128,1,float16,fp8,0,0.10600533088048299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,1,128,1,float16,float16,0,0.0745600014925003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,1,128,1,fp8,fp8,0,0.06487466891606648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,2,128,1,float16,float16,0,0.07590400179227193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,2,128,1,float16,fp8,0,0.07537066439787547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,48,128,1,fp8,fp8,0,0.08778132994969685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,2,128,1,fp8,fp8,0,0.06557866434256236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,4,128,1,float16,float16,0,0.08217599987983704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,4,128,1,float16,fp8,0,0.0782293329636256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,8,128,1,float16,float16,0,0.07996266583601634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,8,128,1,float16,fp8,0,0.07970133423805237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,8,128,1,fp8,fp8,0,0.07506666580835979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,48,4,128,1,fp8,fp8,0,0.07179200152556102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,48,128,1,float16,float16,0,0.05958933134873708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,48,128,1,float16,fp8,0,0.05628266433874766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,48,128,1,fp8,fp8,0,0.06345599889755249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,1,128,1,float16,float16,0,0.04253333310286204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,1,128,1,fp8,fp8,0,0.03821333249409994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,1,128,1,float16,fp8,0,0.04233600199222565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,2,128,1,float16,float16,0,0.04340800146261851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,2,128,1,fp8,fp8,0,0.03877866764863332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,4,128,1,fp8,fp8,0,0.042693331837654114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,4,128,1,float16,float16,0,0.04478933413823446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,4,128,1,float16,fp8,0,0.044778664906819664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,8,128,1,float16,float16,0,0.04540266593297323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,2,128,1,float16,fp8,0,0.043247997760772705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,8,128,1,fp8,fp8,0,0.04322133461634318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,1,128,1,float16,float16,0,0.02995733420054118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,48,128,1,float16,float16,0,0.033071999748547874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,1,128,1,float16,fp8,0,0.030234667162100475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,48,128,1,float16,fp8,0,0.03266666581233343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,48,8,128,1,float16,fp8,0,0.04534400006135305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,48,128,1,fp8,fp8,0,0.030618667602539062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,1,128,1,fp8,fp8,0,0.026917333404223125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,2,128,1,float16,float16,0,0.029839999973773956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,2,128,1,float16,fp8,0,0.030037333567937214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,2,128,1,fp8,fp8,0,0.02739733209212621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,4,128,1,float16,fp8,0,0.030645333230495453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,8,128,1,float16,float16,0,0.030640001098314922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,4,128,1,float16,float16,0,0.030640001098314922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,4,128,1,fp8,fp8,0,0.029045333464940388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,48,128,1,float16,float16,0,0.02294933299223582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,8,128,1,fp8,fp8,0,0.029088000456492107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,48,128,1,float16,fp8,0,0.022874665757020313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,48,128,1,fp8,fp8,0,0.022810667753219604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,48,8,128,1,float16,fp8,0,0.030997333427270252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,1,128,1,float16,float16,0,0.021194666624069214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,1,128,1,float16,fp8,0,0.021402666966120403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,2,128,1,float16,float16,0,0.021397332350413006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,1,128,1,fp8,fp8,0,0.0204373337328434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,2,128,1,float16,fp8,0,0.021674667795499165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,4,128,1,float16,float16,0,0.02204799900452296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,4,128,1,fp8,fp8,0,0.021829334398110706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,2,128,1,fp8,fp8,0,0.020853333175182343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,8,128,1,float16,fp8,0,0.02197866638501485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,8,128,1,float16,float16,0,0.0220320001244545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,4,128,1,float16,fp8,0,0.022053333620230358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,48,8,128,1,fp8,fp8,0,0.02163733293612798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,48,128,1,float16,float16,0,0.018298666924238205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,48,128,1,float16,fp8,0,0.018383999665578205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,48,128,1,fp8,fp8,0,0.018805333723624546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,1,128,1,float16,fp8,0,0.01773333301146825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,1,128,1,fp8,fp8,0,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,1,128,1,float16,float16,0,0.017231999586025875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,2,128,1,float16,fp8,0,0.01752000053723653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,2,128,1,fp8,fp8,0,0.01801066721479098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,4,128,1,float16,float16,0,0.01749333366751671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,2,128,1,float16,float16,0,0.01754133279124896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,4,128,1,float16,fp8,0,0.017845333864291508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,4,128,1,fp8,fp8,0,0.01814933369557063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,8,128,1,float16,float16,0,0.017674667139848072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,8,128,1,float16,fp8,0,0.017984000345071156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,48,8,128,1,fp8,fp8,0,0.017952000101407368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,1,128,1,float16,float16,0,0.5553866624832153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,1,128,1,float16,fp8,0,0.5556639830271403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,2,128,1,float16,fp8,0,0.6635786692301432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,2,128,1,fp8,fp8,0,0.5914026498794556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,2,128,1,float16,float16,0,0.5647733211517334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,1,128,1,fp8,fp8,0,0.5664586623509725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,4,128,1,float16,fp8,0,0.5837813218434652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,4,128,1,float16,float16,0,0.591482679049174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,1,128,1,float16,float16,0,0.2854986588160197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,4,128,1,fp8,fp8,0,0.6559893290201823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,8,128,1,fp8,fp8,0,0.668234666188558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,8,128,1,float16,fp8,0,0.7330719629923502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,48,8,128,1,float16,float16,0,0.5997759898503622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,48,128,1,float16,float16,0,0.38068799177805585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,48,128,1,fp8,fp8,0,0.36615999539693195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,1,128,1,float16,fp8,0,0.28537599245707196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,48,128,1,float16,fp8,0,0.36586666107177734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,1,128,1,fp8,fp8,0,0.29346134265263873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,2,128,1,float16,float16,0,0.29026132822036743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,2,128,1,fp8,fp8,0,0.3009120027224223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,2,128,1,float16,fp8,0,0.3408426841100057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,4,128,1,float16,float16,0,0.30193599065144855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,4,128,1,float16,fp8,0,0.2993386586507161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,4,128,1,fp8,fp8,0,0.33391467730204266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,8,128,1,float16,fp8,0,0.30218666791915894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,8,128,1,float16,float16,0,0.30505067110061646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,48,8,128,1,fp8,fp8,0,0.3403253157933553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,48,128,1,float16,fp8,0,0.2807520031929016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,48,128,1,float16,float16,0,0.20039467016855875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,1,128,1,float16,float16,0,0.15127467115720114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,1,128,1,float16,fp8,0,0.1527359982331594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,2,128,1,float16,fp8,0,0.1527786652247111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,1,128,1,fp8,fp8,0,0.15551466743151346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,2,128,1,float16,float16,0,0.15421332915623984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,2,128,1,fp8,fp8,0,0.1642400026321411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,4,128,1,float16,float16,0,0.15983999768892923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,48,128,1,fp8,fp8,0,0.1919040083885193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,4,128,1,float16,fp8,0,0.15824533502260843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,4,128,1,fp8,fp8,0,0.17242133617401123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,8,128,1,float16,float16,0,0.1629706621170044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,8,128,1,float16,fp8,0,0.16061866283416748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,48,128,1,float16,float16,0,0.15041066209475198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,1,128,1,float16,fp8,0,0.08251733581225078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,48,8,128,1,fp8,fp8,0,0.17648533980051676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,1,128,1,float16,float16,0,0.08164266745249431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,48,128,1,float16,fp8,0,0.10221333305040996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,1,128,1,fp8,fp8,0,0.08730666836102803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,48,128,1,fp8,fp8,0,0.1049066682656606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,2,128,1,float16,float16,0,0.08292266726493835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,2,128,1,float16,fp8,0,0.08627200126647949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,4,128,1,float16,fp8,0,0.08496000369389851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,2,128,1,fp8,fp8,0,0.08567999800046285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,8,128,1,float16,float16,0,0.08737066388130188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,4,128,1,fp8,fp8,0,0.0937546690305074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,4,128,1,float16,float16,0,0.08567999800046285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,8,128,1,fp8,fp8,0,0.09595732887585957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,48,8,128,1,float16,fp8,0,0.08661333719889323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,48,128,1,float16,float16,0,0.07477866609891255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,1,128,1,float16,float16,0,0.047135998805363975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,48,128,1,float16,fp8,0,0.05643199880917867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,1,128,1,float16,fp8,0,0.04678933322429657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,2,128,1,float16,float16,0,0.04744533201058706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,1,128,1,fp8,fp8,0,0.04844800134499868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,48,128,1,fp8,fp8,0,0.060592000683148704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,2,128,1,float16,fp8,0,0.0476746658484141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,2,128,1,fp8,fp8,0,0.04894933104515076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,4,128,1,float16,float16,0,0.04978133241335551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,4,128,1,float16,fp8,0,0.0484746644894282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,4,128,1,fp8,fp8,0,0.05301866432030996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,8,128,1,float16,float16,0,0.05031999945640564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,8,128,1,fp8,fp8,0,0.05339199801286062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,48,128,1,float16,fp8,0,0.031328000128269196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,48,8,128,1,float16,fp8,0,0.05012266834576925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,48,128,1,fp8,fp8,0,0.03429866582155228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,48,128,1,float16,float16,0,0.031680000325044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,1,128,1,float16,float16,0,0.029605334003766377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,1,128,1,float16,fp8,0,0.029711998999118805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,1,128,1,fp8,fp8,0,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,2,128,1,float16,fp8,0,0.030069333811601002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,2,128,1,float16,float16,0,0.029866665601730347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,4,128,1,float16,float16,0,0.030432000756263733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,4,128,1,fp8,fp8,0,0.03944533318281174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,8,128,1,float16,fp8,0,0.031184000273545582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,4,128,1,float16,fp8,0,0.030789333085219067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,2,128,1,fp8,fp8,0,0.030965333183606465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,8,128,1,float16,float16,0,0.03068266560633977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,48,8,128,1,fp8,fp8,0,0.03281066566705704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,48,128,1,float16,fp8,0,0.02475200096766154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,48,128,1,float16,float16,0,0.0244159996509552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,1,128,1,float16,float16,0,0.023130667706330616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,48,128,1,fp8,fp8,0,0.025706666211287182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,2,128,1,float16,float16,0,0.023103999594847362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,1,128,1,float16,fp8,0,0.023226665953795116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,2,128,1,float16,fp8,0,0.023797333240509033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,2,128,1,fp8,fp8,0,0.024549332757790882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,1,128,1,fp8,fp8,0,0.023893333971500397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,4,128,1,float16,fp8,0,0.02383466561635335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,4,128,1,float16,float16,0,0.023733332753181458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,4,128,1,fp8,fp8,0,0.025231999655564625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,8,128,1,float16,float16,0,0.0234400009115537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,8,128,1,fp8,fp8,0,0.02518933266401291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,48,128,1,float16,float16,0,0.017594666530688603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,48,128,1,fp8,fp8,0,0.018858666221300762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,48,8,128,1,float16,fp8,0,0.02401600033044815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,1,128,1,float16,float16,0,0.01626666635274887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,1,128,1,float16,fp8,0,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,48,128,1,float16,fp8,0,0.017711999515692394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,1,128,1,fp8,fp8,0,0.017535999417304993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,2,128,1,float16,float16,0,0.016623999923467636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,2,128,1,float16,fp8,0,0.01681600014368693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,2,128,1,fp8,fp8,0,0.017802666872739792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,4,128,1,float16,fp8,0,0.016704000532627106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,4,128,1,fp8,fp8,0,0.017914666483799618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,4,128,1,float16,float16,0,0.016544000556071598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,8,128,1,fp8,fp8,0,0.017893332988023758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,8,128,1,float16,fp8,0,0.0170666662355264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,48,128,1,float16,float16,0,0.01584533353646596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,48,128,1,float16,fp8,0,0.01621866722901662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,48,8,128,1,float16,float16,0,0.01674666628241539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,1,128,1,float16,float16,0,0.015583999454975128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,48,128,1,fp8,fp8,0,0.017050666113694508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,1,128,1,fp8,fp8,0,0.017024000485738117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,2,128,1,float16,fp8,0,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,2,128,1,float16,float16,0,0.015775999675194424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,1,128,1,float16,fp8,0,0.01616000011563301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,2,128,1,fp8,fp8,0,0.016832000265518825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,4,128,1,float16,fp8,0,0.01587733378012975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,4,128,1,float16,float16,0,0.015743999431530636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,8,128,1,float16,float16,0,0.01573866605758667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,4,128,1,fp8,fp8,0,0.01687466725707054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,8,128,1,float16,fp8,0,0.016271999726692837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,48,8,128,1,fp8,fp8,0,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,1,128,1,float16,fp8,0,0.3840320110321045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,1,128,1,fp8,fp8,0,0.44873066743214923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,2,128,1,float16,fp8,0,0.4339359998703003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,4,128,1,float16,float16,0,0.3982079823811849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,1,128,1,float16,float16,0,0.38412264982859295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,2,128,1,float16,float16,0,0.3877386649449666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,2,128,1,fp8,fp8,0,0.45949868361155194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,4,128,1,float16,fp8,0,0.3982613484064738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,1,128,1,float16,float16,0,0.19987734158833823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,4,128,1,fp8,fp8,0,0.490506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,8,128,1,float16,float16,0,0.40378665924072266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,8,128,1,float16,fp8,0,0.4561386505762736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,48,8,128,1,fp8,fp8,0,0.5012960036595663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,48,128,1,float16,float16,0,0.24195200204849243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,48,128,1,fp8,fp8,0,0.26975466807683307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,1,128,1,fp8,fp8,0,0.23679467042287192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,48,128,1,float16,fp8,0,0.23477866252263388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,2,128,1,float16,float16,0,0.20071999231974283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,1,128,1,float16,fp8,0,0.20034132401148477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,2,128,1,float16,fp8,0,0.22356265783309937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,2,128,1,fp8,fp8,0,0.24062933524449667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,4,128,1,float16,float16,0,0.20634132623672485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,4,128,1,float16,fp8,0,0.20747733116149902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,4,128,1,fp8,fp8,0,0.2525493303934733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,8,128,1,float16,fp8,0,0.2092906634012858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,48,128,1,float16,float16,0,0.16284799575805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,8,128,1,float16,float16,0,0.21098132928212485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,48,128,1,float16,fp8,0,0.12358400225639343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,1,128,1,float16,float16,0,0.10485866665840149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,1,128,1,float16,fp8,0,0.10429867108662923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,48,128,1,fp8,fp8,0,0.14478400349617004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,48,8,128,1,fp8,fp8,0,0.25755200783411664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,1,128,1,fp8,fp8,0,0.1265066663424174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,2,128,1,float16,float16,0,0.10817066828409831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,2,128,1,fp8,fp8,0,0.12893333037694296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,4,128,1,float16,float16,0,0.1072266697883606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,4,128,1,float16,fp8,0,0.10692266623179118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,4,128,1,fp8,fp8,0,0.13492266337076822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,2,128,1,float16,fp8,0,0.10427733262379964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,8,128,1,float16,float16,0,0.10948266585667928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,8,128,1,fp8,fp8,0,0.14076800147692362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,48,8,128,1,float16,fp8,0,0.10909333825111389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,1,128,1,float16,float16,0,0.05815466741720835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,48,128,1,float16,float16,0,0.06927466889222463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,48,128,1,fp8,fp8,0,0.08081066608428955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,48,128,1,float16,fp8,0,0.06714666883150737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,2,128,1,float16,float16,0,0.05810666580994924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,1,128,1,float16,fp8,0,0.05755199988683065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,1,128,1,fp8,fp8,0,0.06944533189137776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,2,128,1,fp8,fp8,0,0.0703359991312027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,4,128,1,float16,float16,0,0.059119999408721924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,2,128,1,float16,fp8,0,0.05914133290449778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,4,128,1,float16,fp8,0,0.0594400018453598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,8,128,1,float16,float16,0,0.06058666606744131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,4,128,1,fp8,fp8,0,0.07352533439795177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,8,128,1,float16,fp8,0,0.06060799956321716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,48,128,1,float16,float16,0,0.036144000788529716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,1,128,1,float16,float16,0,0.034703999757766724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,48,128,1,float16,fp8,0,0.03639466563860575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,48,128,1,fp8,fp8,0,0.044591998060544334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,48,8,128,1,fp8,fp8,0,0.07421333094437917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,1,128,1,float16,fp8,0,0.03499733408292135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,2,128,1,float16,fp8,0,0.034927998979886375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,1,128,1,fp8,fp8,0,0.04061333338419596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,4,128,1,float16,float16,0,0.035349334279696144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,2,128,1,fp8,fp8,0,0.04098666707674662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,4,128,1,float16,fp8,0,0.03532800078392029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,4,128,1,fp8,fp8,0,0.042837331692377724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,8,128,1,float16,float16,0,0.03589866558710734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,2,128,1,float16,float16,0,0.0347626656293869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,8,128,1,fp8,fp8,0,0.04298133154710134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,48,8,128,1,float16,fp8,0,0.03572266548871994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,48,128,1,float16,float16,0,0.024800000091393787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,48,128,1,float16,fp8,0,0.02502399931351344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,48,128,1,fp8,fp8,0,0.029653333127498627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,1,128,1,float16,fp8,0,0.02385066697994868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,1,128,1,float16,float16,0,0.023711999257405598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,2,128,1,float16,float16,0,0.02380266785621643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,2,128,1,fp8,fp8,0,0.027658666173617046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,4,128,1,float16,fp8,0,0.024298667907714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,4,128,1,float16,float16,0,0.024069334069887798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,2,128,1,float16,fp8,0,0.02383466561635335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,4,128,1,fp8,fp8,0,0.028783999383449554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,1,128,1,fp8,fp8,0,0.027632000545660656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,8,128,1,float16,float16,0,0.02404266595840454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,8,128,1,fp8,fp8,0,0.02924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,1,128,1,float16,float16,0,0.01860800012946129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,48,128,1,float16,float16,0,0.01939733326435089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,48,8,128,1,float16,fp8,0,0.024485332270463307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,48,128,1,float16,fp8,0,0.01934933289885521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,1,128,1,float16,fp8,0,0.01844266677896182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,48,128,1,fp8,fp8,0,0.021903999149799347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,1,128,1,fp8,fp8,0,0.021130666136741638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,2,128,1,float16,float16,0,0.018687999496857326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,2,128,1,float16,fp8,0,0.019039999693632126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,2,128,1,fp8,fp8,0,0.021269333859284718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,4,128,1,float16,float16,0,0.01817600056529045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,4,128,1,float16,fp8,0,0.01874133323629697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,8,128,1,float16,float16,0,0.018661333868900936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,4,128,1,fp8,fp8,0,0.021642667551835377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,8,128,1,float16,fp8,0,0.019167999426523846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,48,8,128,1,fp8,fp8,0,0.021317332983016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,48,128,1,float16,float16,0,0.015381333728631338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,48,128,1,float16,fp8,0,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,48,128,1,fp8,fp8,0,0.017765333255132038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,1,128,1,float16,fp8,0,0.015386667102575302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,1,128,1,float16,float16,0,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,2,128,1,float16,float16,0,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,1,128,1,fp8,fp8,0,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,2,128,1,float16,fp8,0,0.015482666591803232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,2,128,1,fp8,fp8,0,0.01754666616519292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,4,128,1,float16,float16,0,0.015002666662136713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,4,128,1,float16,fp8,0,0.015418666104475657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,4,128,1,fp8,fp8,0,0.01729600007335345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,8,128,1,float16,fp8,0,0.015530666957298914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,8,128,1,float16,float16,0,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,48,128,1,float16,float16,0,0.01462399959564209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,48,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,48,128,1,fp8,fp8,0,0.016544000556071598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,48,8,128,1,fp8,fp8,0,0.019946667055288952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,1,128,1,float16,float16,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,1,128,1,float16,fp8,0,0.015130666395028433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,1,128,1,fp8,fp8,0,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,2,128,1,float16,fp8,0,0.01540800059835116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,2,128,1,float16,float16,0,0.01458666721979777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,2,128,1,fp8,fp8,0,0.016623999923467636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,4,128,1,float16,float16,0,0.014453332871198654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,4,128,1,float16,fp8,0,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,4,128,1,fp8,fp8,0,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,8,128,1,float16,float16,0,0.014490666488806406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,8,128,1,float16,fp8,0,0.015114666273196539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,48,8,128,1,fp8,fp8,0,0.016607999801635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,1,128,1,float16,float16,0,0.32705066601435345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,1,128,1,fp8,fp8,0,0.3882879813512166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,1,128,1,float16,fp8,0,0.34114666779836017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,2,128,1,fp8,fp8,0,0.39155733585357666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,4,128,1,float16,float16,0,0.3309173385302226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,2,128,1,float16,float16,0,0.32683734099070233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,4,128,1,float16,fp8,0,0.33126399914423627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,2,128,1,float16,fp8,0,0.3266666730244954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,4,128,1,fp8,fp8,0,0.406272013982137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,8,128,1,float16,float16,0,0.33424532413482666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,8,128,1,float16,fp8,0,0.3514346679051717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,48,8,128,1,fp8,fp8,0,0.4115626811981201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,48,128,1,float16,float16,0,0.1769333283106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,1,128,1,float16,fp8,0,0.16921067237854004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,1,128,1,fp8,fp8,0,0.20382400353749594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,48,128,1,float16,fp8,0,0.17365866899490356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,48,128,1,fp8,fp8,0,0.2207146684328715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,1,128,1,float16,float16,0,0.17005334297815958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,2,128,1,float16,float16,0,0.17172267039616904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,2,128,1,float16,fp8,0,0.16858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,2,128,1,fp8,fp8,0,0.2064746618270874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,4,128,1,float16,fp8,0,0.17137600978215536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,4,128,1,float16,float16,0,0.17188799381256104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,8,128,1,float16,float16,0,0.17415465911229452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,4,128,1,fp8,fp8,0,0.2103360096613566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,8,128,1,float16,fp8,0,0.17451733350753784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,48,8,128,1,fp8,fp8,0,0.21667200326919556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,48,128,1,float16,float16,0,0.09422933061917622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,1,128,1,float16,float16,0,0.08972799777984619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,1,128,1,float16,fp8,0,0.0915786623954773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,48,128,1,float16,fp8,0,0.09160533547401428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,48,128,1,fp8,fp8,0,0.11912533640861511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,2,128,1,float16,fp8,0,0.09212266405423482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,2,128,1,float16,float16,0,0.0913706620534261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,1,128,1,fp8,fp8,0,0.10939733187357585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,2,128,1,fp8,fp8,0,0.10954667131106059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,4,128,1,float16,float16,0,0.09220799803733826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,4,128,1,float16,fp8,0,0.09259733557701111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,8,128,1,float16,float16,0,0.0921013355255127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,4,128,1,fp8,fp8,0,0.11433066924413045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,8,128,1,float16,fp8,0,0.09353066484133403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,48,128,1,float16,fp8,0,0.048309331138928734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,48,128,1,fp8,fp8,0,0.06470933556556702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,1,128,1,float16,float16,0,0.0518453319867452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,1,128,1,float16,fp8,0,0.05141866703828176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,1,128,1,fp8,fp8,0,0.06205333272616068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,48,8,128,1,fp8,fp8,0,0.11355200409889221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,48,128,1,float16,float16,0,0.04942933221658071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,2,128,1,float16,float16,0,0.05157333115736643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,2,128,1,float16,fp8,0,0.05212266743183136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,4,128,1,float16,float16,0,0.05316799879074097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,2,128,1,fp8,fp8,0,0.06260266900062561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,4,128,1,float16,fp8,0,0.05263466636339823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,4,128,1,fp8,fp8,0,0.06389866769313812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,8,128,1,float16,float16,0,0.052970667680104576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,8,128,1,fp8,fp8,0,0.06393600006898244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,48,128,1,float16,float16,0,0.031930667658646904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,1,128,1,float16,fp8,0,0.03244800120592117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,48,128,1,float16,fp8,0,0.03199466566244761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,48,8,128,1,float16,fp8,0,0.0529120018084844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,1,128,1,float16,float16,0,0.03216533362865448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,48,128,1,fp8,fp8,0,0.040218666195869446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,2,128,1,float16,float16,0,0.03236799935499827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,2,128,1,fp8,fp8,0,0.03868266691764196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,2,128,1,float16,fp8,0,0.0324799989660581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,4,128,1,float16,float16,0,0.03296533226966858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,1,128,1,fp8,fp8,0,0.038880000511805214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,4,128,1,fp8,fp8,0,0.039247999588648476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,8,128,1,float16,fp8,0,0.03301866600910822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,4,128,1,float16,fp8,0,0.033189333975315094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,8,128,1,fp8,fp8,0,0.039893334110577904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,48,8,128,1,float16,float16,0,0.03294933338960012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,48,128,1,float16,fp8,0,0.021856000026067097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,48,128,1,float16,float16,0,0.021344001094500225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,48,128,1,fp8,fp8,0,0.025648000339667004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,1,128,1,float16,fp8,0,0.021002667645613354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,2,128,1,fp8,fp8,0,0.025237334271272022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,2,128,1,float16,float16,0,0.02165866643190384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,2,128,1,float16,fp8,0,0.022101332743962605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,4,128,1,float16,float16,0,0.021189334491888683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,1,128,1,fp8,fp8,0,0.02531733363866806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,1,128,1,float16,float16,0,0.021157334248224895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,4,128,1,float16,fp8,0,0.021664001047611237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,4,128,1,fp8,fp8,0,0.025701334079106648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,8,128,1,float16,float16,0,0.021205333371957142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,8,128,1,float16,fp8,0,0.02139200021823247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,1,128,1,fp8,fp8,0,0.020970667401949566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,1,128,1,float16,fp8,0,0.018112000077962875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,2,128,1,float16,float16,0,0.017594666530688603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,48,8,128,1,fp8,fp8,0,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,48,128,1,float16,float16,0,0.017658667018016178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,48,128,1,fp8,fp8,0,0.02077866718173027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,1,128,1,float16,float16,0,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,48,128,1,float16,fp8,0,0.018320000420014065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,2,128,1,float16,fp8,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,4,128,1,float16,float16,0,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,2,128,1,fp8,fp8,0,0.021055998901526134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,4,128,1,float16,fp8,0,0.018197332819302876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,4,128,1,fp8,fp8,0,0.02130666623512904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,8,128,1,float16,float16,0,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,8,128,1,float16,fp8,0,0.01844800015290578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,48,128,1,float16,float16,0,0.014661333213249842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,48,128,1,float16,fp8,0,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,48,128,1,fp8,fp8,0,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,1,128,1,float16,float16,0,0.014602666099866232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,1,128,1,float16,fp8,0,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,1,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,48,8,128,1,fp8,fp8,0,0.021040000021457672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,2,128,1,float16,fp8,0,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,2,128,1,fp8,fp8,0,0.016869333883126576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,4,128,1,float16,float16,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,2,128,1,float16,float16,0,0.014229333649079004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,4,128,1,float16,fp8,0,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,8,128,1,float16,float16,0,0.014666666587193808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,4,128,1,fp8,fp8,0,0.017055999487638474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,8,128,1,fp8,fp8,0,0.01759999990463257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,48,8,128,1,float16,fp8,0,0.01552533358335495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,48,128,1,float16,float16,0,0.014085333794355392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,48,128,1,float16,fp8,0,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,1,128,1,float16,fp8,0,0.014842666685581207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,1,128,1,float16,float16,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,1,128,1,fp8,fp8,0,0.016127999871969223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,48,128,1,fp8,fp8,0,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,2,128,1,float16,float16,0,0.014298666268587112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,2,128,1,fp8,fp8,0,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,4,128,1,float16,fp8,0,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,8,128,1,float16,float16,0,0.014165333161751429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,4,128,1,float16,float16,0,0.0143306665122509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,4,128,1,fp8,fp8,0,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,2,128,1,float16,fp8,0,0.014720000326633453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,8,128,1,fp8,fp8,0,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,48,8,128,1,float16,fp8,0,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,1,128,1,fp8,fp8,0,26.842740376790363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,1,128,1,float16,float16,0,41.116292317708336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,1,128,1,float16,fp8,0,40.04610697428385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,2,128,1,fp8,fp8,0,26.41553497314453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,2,128,1,float16,float16,0,41.734031677246094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,2,128,1,float16,fp8,0,40.83509318033854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,4,128,1,float16,float16,0,41.261924743652344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,4,128,1,float16,fp8,0,40.74804178873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,1,128,1,float16,float16,0,20.26909891764323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,4,128,1,fp8,fp8,0,26.88213857014974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,8,128,1,float16,float16,0,41.019360860188804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,8,128,1,fp8,fp8,0,26.584261576334637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,40,128,1,float16,float16,0,20.687989552815754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,40,8,128,1,float16,fp8,0,40.50619761149088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,40,128,1,float16,fp8,0,20.26341374715169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,1,128,1,float16,fp8,0,20.475706736246746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,1,128,1,fp8,fp8,0,13.588619232177734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,40,128,1,fp8,fp8,0,13.257514953613281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,2,128,1,float16,float16,0,20.698346455891926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,2,128,1,fp8,fp8,0,13.527200063069662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,2,128,1,float16,fp8,0,20.373274485270183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,4,128,1,float16,float16,0,20.16438929239909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,4,128,1,float16,fp8,0,20.503028869628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,8,128,1,float16,float16,0,20.796133677164715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,4,128,1,fp8,fp8,0,13.331631978352865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,8,128,1,float16,fp8,0,20.703243255615234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,40,8,128,1,fp8,fp8,0,13.398085276285807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,1,128,1,fp8,fp8,0,6.602176030476888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,1,128,1,float16,fp8,0,10.596314748128256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,1,128,1,float16,float16,0,10.35264523824056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,2,128,1,float16,float16,0,10.43345578511556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,40,128,1,float16,float16,0,10.36793581644694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,2,128,1,float16,fp8,0,10.311930974324545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,2,128,1,fp8,fp8,0,6.613061269124349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,40,128,1,fp8,fp8,0,6.624794642130534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,4,128,1,float16,float16,0,10.587397257486979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,4,128,1,fp8,fp8,0,6.813146591186523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,40,128,1,float16,fp8,0,10.249754587809244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,4,128,1,float16,fp8,0,10.322421391805014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,8,128,1,float16,float16,0,10.503434499104818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,1,128,1,float16,float16,0,5.081040064493815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,1,128,1,float16,fp8,0,5.287535985310872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,1,128,1,fp8,fp8,0,3.424549420674642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,8,128,1,float16,fp8,0,10.48368008931478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,40,128,1,float16,float16,0,4.929973284403483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,40,128,1,float16,fp8,0,4.981760025024414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,40,8,128,1,fp8,fp8,0,6.740959803263347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,40,128,1,fp8,fp8,0,3.4043146769205728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,2,128,1,float16,float16,0,5.069247881571452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,2,128,1,float16,fp8,0,5.157055854797363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,2,128,1,fp8,fp8,0,3.4221226374308267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,4,128,1,float16,float16,0,5.283712069193522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,4,128,1,float16,fp8,0,5.117530822753906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,4,128,1,fp8,fp8,0,3.4304958979288735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,8,128,1,float16,fp8,0,5.183594703674316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,8,128,1,float16,float16,0,5.096090634663899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,40,8,128,1,fp8,fp8,0,3.432133356730143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,1,128,1,float16,float16,0,23.872042338053387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,1,128,1,float16,fp8,0,23.841700236002605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,1,128,1,fp8,fp8,0,15.505391438802084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,2,128,1,fp8,fp8,0,15.52560551961263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,2,128,1,float16,float16,0,23.396507263183594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,2,128,1,float16,fp8,0,24.1229985555013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,4,128,1,float16,float16,0,23.730784098307293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,4,128,1,float16,fp8,0,23.676793416341145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,4,128,1,fp8,fp8,0,15.657760620117188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,1,128,1,float16,float16,0,11.99936040242513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,8,128,1,float16,float16,0,24.38959503173828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,8,128,1,float16,fp8,0,23.6779047648112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,40,8,128,1,fp8,fp8,0,15.687039693196615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,40,128,1,float16,fp8,0,11.985706329345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,40,128,1,float16,float16,0,11.75445302327474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,1,128,1,float16,fp8,0,11.863343556722006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,1,128,1,fp8,fp8,0,7.876485188802083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,2,128,1,float16,float16,0,12.161407470703125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,40,128,1,fp8,fp8,0,7.857957204182942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,2,128,1,float16,fp8,0,12.002304077148438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,2,128,1,fp8,fp8,0,7.922864278157552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,4,128,1,float16,float16,0,12.17629877726237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,4,128,1,fp8,fp8,0,7.792106628417969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,4,128,1,float16,fp8,0,11.797616322835287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,8,128,1,float16,float16,0,11.944960276285807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,1,128,1,float16,float16,0,5.895914713541667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,8,128,1,float16,fp8,0,12.116053263346354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,40,128,1,float16,float16,0,5.95350964864095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,1,128,1,float16,fp8,0,5.7619679768880205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,1,128,1,fp8,fp8,0,3.891162554423014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,40,8,128,1,fp8,fp8,0,7.800346374511719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,40,128,1,float16,fp8,0,5.983754475911458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,2,128,1,float16,float16,0,5.843733469645183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,40,128,1,fp8,fp8,0,3.983344078063965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,2,128,1,float16,fp8,0,5.865845362345378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,2,128,1,fp8,fp8,0,3.8660106658935547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,4,128,1,float16,float16,0,5.81932258605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,4,128,1,fp8,fp8,0,3.904032071431478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,4,128,1,float16,fp8,0,5.991941452026367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,8,128,1,float16,float16,0,6.097583770751953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,8,128,1,float16,fp8,0,6.041802724202474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,1,128,1,float16,fp8,0,2.958826700846354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,1,128,1,float16,float16,0,2.9930187861124673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,1,128,1,fp8,fp8,0,2.065861384073893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,40,8,128,1,fp8,fp8,0,3.906010627746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,40,128,1,float16,float16,0,2.9900051752726235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,40,128,1,float16,fp8,0,2.9444052378336587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,40,128,1,fp8,fp8,0,2.0506505966186523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,2,128,1,float16,float16,0,3.0065972010294595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,2,128,1,float16,fp8,0,2.9527200063069663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,2,128,1,fp8,fp8,0,2.0664745966593423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,4,128,1,float16,float16,0,2.983210563659668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,4,128,1,float16,fp8,0,3.001941363016764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,4,128,1,fp8,fp8,0,2.0811840693155923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,8,128,1,float16,float16,0,3.0000425974527993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,8,128,1,fp8,fp8,0,2.0789333979288735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,40,8,128,1,float16,fp8,0,2.976618766784668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,1,128,1,fp8,fp8,0,11.126239776611328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,1,128,1,float16,float16,0,16.935365041097004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,1,128,1,float16,fp8,0,16.800064086914062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,2,128,1,float16,float16,0,17.179738362630207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,2,128,1,float16,fp8,0,17.041717529296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,2,128,1,fp8,fp8,0,11.17563247680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,4,128,1,float16,fp8,0,16.599381764729817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,4,128,1,float16,float16,0,16.859882354736328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,1,128,1,float16,float16,0,8.376346588134766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,4,128,1,fp8,fp8,0,11.167616526285807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,8,128,1,fp8,fp8,0,11.39749272664388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,1,128,1,float16,fp8,0,8.5087521870931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,8,128,1,float16,float16,0,16.868746439615887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,40,8,128,1,float16,fp8,0,16.80526351928711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,40,128,1,float16,float16,0,8.690560022989908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,40,128,1,float16,fp8,0,8.364394505818685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,40,128,1,fp8,fp8,0,5.486885070800781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,1,128,1,fp8,fp8,0,5.3706614176432295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,2,128,1,float16,float16,0,8.48531723022461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,2,128,1,float16,fp8,0,8.47161610921224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,4,128,1,float16,float16,0,8.363173166910807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,2,128,1,fp8,fp8,0,5.561194737752278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,4,128,1,float16,fp8,0,8.539093017578125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,4,128,1,fp8,fp8,0,5.459871927897136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,8,128,1,float16,float16,0,8.386720021565756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,8,128,1,float16,fp8,0,8.282911936442057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,1,128,1,float16,float16,0,4.105242729187012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,1,128,1,float16,fp8,0,4.025717417399089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,1,128,1,fp8,fp8,0,2.78985595703125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,40,128,1,float16,float16,0,4.042970657348633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,2,128,1,float16,float16,0,4.116922696431478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,40,8,128,1,fp8,fp8,0,5.457455952962239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,2,128,1,float16,fp8,0,4.245120048522949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,2,128,1,fp8,fp8,0,2.785546620686849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,40,128,1,fp8,fp8,0,2.8103466033935547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,40,128,1,float16,fp8,0,3.9828532536824546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,4,128,1,float16,float16,0,4.129749298095703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,4,128,1,fp8,fp8,0,2.793370564778646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,4,128,1,float16,fp8,0,4.184368133544922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,8,128,1,float16,float16,0,4.144346555074056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,8,128,1,float16,fp8,0,4.113658587137858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,40,8,128,1,fp8,fp8,0,2.8071680068969727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,1,128,1,float16,float16,0,2.155855973561605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,1,128,1,float16,fp8,0,2.1344587008158364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,40,128,1,float16,float16,0,2.134533405303955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,40,128,1,float16,fp8,0,2.10697603225708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,2,128,1,float16,float16,0,2.1707305908203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,1,128,1,fp8,fp8,0,1.5230666796366374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,2,128,1,float16,fp8,0,2.139791965484619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,40,128,1,fp8,fp8,0,1.5185227394104004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,2,128,1,fp8,fp8,0,1.5167892773946126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,4,128,1,float16,float16,0,2.1662346522013345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,4,128,1,float16,fp8,0,2.1377013524373374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,4,128,1,fp8,fp8,0,1.5200533866882324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,8,128,1,float16,float16,0,2.1822400093078613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,8,128,1,float16,fp8,0,2.1764586766560874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,40,8,128,1,fp8,fp8,0,1.5267465909322102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,1,128,1,fp8,fp8,0,14.707541147867838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,1,128,1,float16,fp8,0,21.930521647135418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,2,128,1,fp8,fp8,0,15.013941446940104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,2,128,1,float16,float16,0,21.95349884033203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,1,128,1,float16,float16,0,21.942230224609375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,4,128,1,float16,float16,0,22.52257537841797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,2,128,1,float16,fp8,0,21.959035237630207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,4,128,1,float16,fp8,0,22.18921661376953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,1,128,1,float16,float16,0,11.237791697184244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,4,128,1,fp8,fp8,0,14.83560053507487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,1,128,1,float16,fp8,0,11.132651011149088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,8,128,1,float16,float16,0,22.3898442586263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,8,128,1,float16,fp8,0,22.499979654947918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,40,128,1,float16,float16,0,10.974255879720053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,40,8,128,1,fp8,fp8,0,14.720778147379557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,40,128,1,fp8,fp8,0,7.565557479858398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,1,128,1,fp8,fp8,0,7.296154657999675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,40,128,1,float16,fp8,0,11.242389678955078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,2,128,1,float16,float16,0,11.118447621663412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,2,128,1,float16,fp8,0,11.148704528808594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,2,128,1,fp8,fp8,0,7.4252268473307295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,4,128,1,float16,fp8,0,11.090080261230469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,4,128,1,float16,float16,0,11.009632110595703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,4,128,1,fp8,fp8,0,7.241205215454102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,8,128,1,float16,float16,0,11.243909200032553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,8,128,1,float16,fp8,0,11.19387690226237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,1,128,1,float16,fp8,0,5.367567698160808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,1,128,1,float16,float16,0,5.303690592447917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,40,8,128,1,fp8,fp8,0,7.454709370930989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,1,128,1,fp8,fp8,0,3.62169615427653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,40,128,1,float16,fp8,0,5.543354670206706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,40,128,1,float16,float16,0,5.356037139892578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,2,128,1,float16,float16,0,5.488293329874675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,2,128,1,float16,fp8,0,5.177776018778483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,2,128,1,fp8,fp8,0,3.6225706736246743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,4,128,1,float16,float16,0,5.354922612508138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,4,128,1,float16,fp8,0,5.430469512939453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,40,128,1,fp8,fp8,0,3.748837471008301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,4,128,1,fp8,fp8,0,3.696277300516764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,8,128,1,float16,float16,0,5.397541046142578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,1,128,1,float16,float16,0,2.7323519388834634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,8,128,1,fp8,fp8,0,3.66155211130778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,1,128,1,float16,fp8,0,2.7224480311075845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,40,8,128,1,float16,fp8,0,5.590080261230469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,1,128,1,fp8,fp8,0,1.9050453503926594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,40,128,1,float16,fp8,0,2.7071892420450845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,40,128,1,fp8,fp8,0,1.9406986236572266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,40,128,1,float16,float16,0,2.708042780558268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,2,128,1,float16,float16,0,2.722346623738607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,2,128,1,float16,fp8,0,2.712640126546224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,2,128,1,fp8,fp8,0,1.9267093340555828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,4,128,1,float16,float16,0,2.769477208455404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,4,128,1,float16,fp8,0,2.6902987162272134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,4,128,1,fp8,fp8,0,1.907589276631673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,8,128,1,float16,float16,0,2.746917406717936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,8,128,1,float16,fp8,0,2.706165313720703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,40,8,128,1,fp8,fp8,0,1.9194879531860352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,1,128,1,float16,float16,0,1.4648853937784831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,40,128,1,float16,float16,0,1.4509600003560383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,40,128,1,float16,fp8,0,1.4632587432861328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,40,128,1,fp8,fp8,0,1.0691893100738525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,1,128,1,float16,fp8,0,1.4491893450419109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,1,128,1,fp8,fp8,0,1.0552639961242676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,2,128,1,float16,float16,0,1.4650239944458008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,2,128,1,float16,fp8,0,1.4482666651407878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,2,128,1,fp8,fp8,0,1.0507840315500896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,4,128,1,float16,float16,0,1.468784014383952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,4,128,1,float16,fp8,0,1.4578132629394531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,4,128,1,fp8,fp8,0,1.0569706757863362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,8,128,1,float16,float16,0,1.4764639536539714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,8,128,1,float16,fp8,0,1.4549652735392253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,40,8,128,1,fp8,fp8,0,1.059717337290446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,1,128,1,float16,float16,0,13.11751937866211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,1,128,1,float16,fp8,0,13.04959487915039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,1,128,1,fp8,fp8,0,8.857242584228516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,2,128,1,float16,float16,0,13.511632283528646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,2,128,1,fp8,fp8,0,8.878421147664389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,4,128,1,float16,float16,0,13.12060292561849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,4,128,1,float16,fp8,0,12.842938741048178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,2,128,1,float16,fp8,0,13.25558344523112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,1,128,1,float16,float16,0,6.663114547729492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,4,128,1,fp8,fp8,0,8.782938639322916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,8,128,1,float16,float16,0,13.289904276529947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,8,128,1,float16,fp8,0,13.139867146809896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,40,128,1,float16,fp8,0,6.519840240478516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,40,128,1,float16,float16,0,6.556501388549805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,1,128,1,float16,fp8,0,6.42579714457194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,40,128,1,fp8,fp8,0,4.466394742329915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,1,128,1,fp8,fp8,0,4.374239921569824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,40,8,128,1,fp8,fp8,0,9.15002695719401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,2,128,1,float16,fp8,0,6.40823491414388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,2,128,1,float16,float16,0,6.477994918823242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,2,128,1,fp8,fp8,0,4.33411184946696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,4,128,1,float16,float16,0,6.467983881632487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,4,128,1,float16,fp8,0,6.5394026438395185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,4,128,1,fp8,fp8,0,4.350986798604329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,8,128,1,float16,float16,0,6.586490631103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,8,128,1,float16,fp8,0,6.563482920328776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,1,128,1,float16,float16,0,3.239546775817871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,40,8,128,1,fp8,fp8,0,4.42362117767334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,1,128,1,float16,fp8,0,3.127824147542318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,40,128,1,float16,float16,0,3.193274815877279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,1,128,1,fp8,fp8,0,2.2211146354675293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,40,128,1,float16,fp8,0,3.1734186808268228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,2,128,1,float16,float16,0,3.2286081314086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,2,128,1,fp8,fp8,0,2.22324800491333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,2,128,1,float16,fp8,0,3.220399856567383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,40,128,1,fp8,fp8,0,2.3179093996683755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,4,128,1,float16,float16,0,3.1726293563842773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,4,128,1,float16,fp8,0,3.1065600713094077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,4,128,1,fp8,fp8,0,2.232463995615641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,8,128,1,float16,float16,0,3.1720800399780273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,8,128,1,float16,fp8,0,3.2307252883911133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,1,128,1,float16,float16,0,1.6458346048990886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,1,128,1,float16,fp8,0,1.6237012545267742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,40,8,128,1,fp8,fp8,0,2.2510453859965005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,40,128,1,float16,fp8,0,1.6488159497578938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,1,128,1,fp8,fp8,0,1.182085355122884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,40,128,1,float16,float16,0,1.6850719451904297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,2,128,1,float16,fp8,0,1.624575932820638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,2,128,1,float16,float16,0,1.6756800015767415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,2,128,1,fp8,fp8,0,1.1847573121388753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,4,128,1,float16,fp8,0,1.6342132886250813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,4,128,1,float16,float16,0,1.6525440216064453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,4,128,1,fp8,fp8,0,1.1881919701894124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,40,128,1,fp8,fp8,0,1.2204960187276204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,8,128,1,float16,fp8,0,1.6342453956604004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,1,128,1,float16,float16,0,0.9093813101450602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,40,128,1,float16,float16,0,0.905029296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,8,128,1,fp8,fp8,0,1.2202186584472656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,1,128,1,float16,fp8,0,0.8923253218332926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,40,8,128,1,float16,float16,0,1.6834774017333984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,40,128,1,fp8,fp8,0,0.6833759943644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,40,128,1,float16,fp8,0,0.8963413238525391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,1,128,1,fp8,fp8,0,0.6688533624013265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,2,128,1,float16,float16,0,0.9099573294321696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,2,128,1,float16,fp8,0,0.894266684850057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,2,128,1,fp8,fp8,0,0.6759040355682373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,4,128,1,float16,float16,0,0.9105707009633383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,4,128,1,fp8,fp8,0,0.6722933451334635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,4,128,1,float16,fp8,0,0.9038346608479818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,8,128,1,float16,float16,0,0.9143626689910889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,8,128,1,float16,fp8,0,0.8977706432342529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,40,8,128,1,fp8,fp8,0,0.6761919657389323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,1,128,1,fp8,fp8,0,8.681573232014975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,1,128,1,float16,float16,0,12.708863576253256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,2,128,1,float16,float16,0,12.825631459554037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,2,128,1,fp8,fp8,0,8.830517450968424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,2,128,1,float16,fp8,0,12.877508799235025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,1,128,1,float16,fp8,0,13.154144287109375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,4,128,1,float16,fp8,0,12.696975708007812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,4,128,1,float16,float16,0,12.885178883870443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,1,128,1,float16,float16,0,6.1349442799886065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,4,128,1,fp8,fp8,0,8.753098805745443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,8,128,1,float16,float16,0,12.935376485188803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,1,128,1,float16,fp8,0,6.140719731648763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,8,128,1,float16,fp8,0,12.816805521647135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,40,8,128,1,fp8,fp8,0,9.07093874613444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,40,128,1,float16,fp8,0,6.807504018147786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,40,128,1,fp8,fp8,0,4.645658810933431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,40,128,1,float16,float16,0,6.757008234659831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,1,128,1,fp8,fp8,0,4.372949282328288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,2,128,1,fp8,fp8,0,4.383978525797526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,2,128,1,float16,float16,0,6.3630828857421875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,2,128,1,float16,fp8,0,6.264149347941081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,4,128,1,float16,float16,0,6.416432062784831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,4,128,1,float16,fp8,0,6.465311686197917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,4,128,1,fp8,fp8,0,4.405381202697754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,1,128,1,float16,fp8,0,2.992938677469889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,1,128,1,float16,float16,0,3.119200070699056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,8,128,1,float16,float16,0,6.350512186686198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,8,128,1,float16,fp8,0,6.134090423583984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,40,8,128,1,fp8,fp8,0,4.450912157694499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,40,128,1,float16,fp8,0,3.125189463297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,40,128,1,float16,float16,0,3.1873706181844077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,1,128,1,fp8,fp8,0,2.217093308766683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,2,128,1,float16,float16,0,3.1264425913492837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,40,128,1,fp8,fp8,0,2.364837328592936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,2,128,1,float16,fp8,0,3.044666608174642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,4,128,1,float16,float16,0,3.052922566731771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,2,128,1,fp8,fp8,0,2.2217226028442383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,4,128,1,fp8,fp8,0,2.233018716176351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,4,128,1,float16,fp8,0,3.0233227411905923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,8,128,1,float16,float16,0,3.092735926310221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,8,128,1,float16,fp8,0,3.1249545415242515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,40,8,128,1,fp8,fp8,0,2.255349318186442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,1,128,1,float16,fp8,0,1.5443092981974285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,1,128,1,float16,float16,0,1.5734826723734539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,40,128,1,float16,float16,0,1.6496213277180989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,1,128,1,fp8,fp8,0,1.1566932996114094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,40,128,1,float16,fp8,0,1.5960052808125813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,2,128,1,float16,float16,0,1.617402712504069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,2,128,1,float16,fp8,0,1.5522027015686035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,2,128,1,fp8,fp8,0,1.1574506759643555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,4,128,1,float16,float16,0,1.5814026196797688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,40,128,1,fp8,fp8,0,1.2354239622751872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,4,128,1,float16,fp8,0,1.5718933741251628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,4,128,1,fp8,fp8,0,1.1649973392486572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,1,128,1,float16,float16,0,0.8490026791890463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,8,128,1,float16,fp8,0,1.6107840538024902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,8,128,1,float16,float16,0,1.5893227259318035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,1,128,1,float16,fp8,0,0.8266932964324951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,40,8,128,1,fp8,fp8,0,1.1785386403401692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,40,128,1,float16,float16,0,0.8563306331634521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,40,128,1,float16,fp8,0,0.8462986946105957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,1,128,1,fp8,fp8,0,0.6301120122273763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,2,128,1,float16,fp8,0,0.8293440341949463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,2,128,1,float16,float16,0,0.8671786785125732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,2,128,1,fp8,fp8,0,0.6325546503067017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,40,128,1,fp8,fp8,0,0.6653973261515299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,4,128,1,float16,fp8,0,0.8305439949035645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,4,128,1,float16,float16,0,0.8490560054779053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,4,128,1,fp8,fp8,0,0.6355093320210775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,8,128,1,float16,float16,0,0.8581439654032389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,8,128,1,float16,fp8,0,0.8361813227335612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,1,128,1,float16,float16,0,0.4824906587600708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,40,128,1,float16,float16,0,0.4829973379770915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,40,8,128,1,fp8,fp8,0,0.6588053305943807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,1,128,1,float16,fp8,0,0.47155733903249103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,40,128,1,fp8,fp8,0,0.375434676806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,1,128,1,fp8,fp8,0,0.3508426745732625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,2,128,1,float16,float16,0,0.4833386739095052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,40,128,1,float16,fp8,0,0.47446401913960773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,2,128,1,float16,fp8,0,0.4729333321253459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,2,128,1,fp8,fp8,0,0.3519839843114217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,4,128,1,float16,float16,0,0.49395732084910077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,4,128,1,float16,fp8,0,0.474181334177653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,4,128,1,fp8,fp8,0,0.35259199142456055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,8,128,1,float16,float16,0,0.4865066607793172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,8,128,1,float16,fp8,0,0.4761759837468465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,40,8,128,1,fp8,fp8,0,0.3572106758753459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,1,128,1,float16,float16,0,7.674842834472656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,1,128,1,float16,fp8,0,7.605669021606445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,2,128,1,float16,fp8,0,7.675402959187825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,2,128,1,fp8,fp8,0,5.576442718505859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,2,128,1,float16,float16,0,7.698127746582031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,1,128,1,fp8,fp8,0,5.927754720052083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,4,128,1,float16,float16,0,7.760271708170573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,4,128,1,float16,fp8,0,7.594229380289714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,1,128,1,float16,float16,0,3.7203359603881836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,4,128,1,fp8,fp8,0,5.564175923665364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,8,128,1,float16,float16,0,7.846047719319661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,8,128,1,float16,fp8,0,7.597914377848308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,40,8,128,1,fp8,fp8,0,5.717589060465495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,1,128,1,float16,fp8,0,3.685114542643229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,40,128,1,fp8,fp8,0,3.0067574183146157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,40,128,1,float16,float16,0,4.023802757263184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,40,128,1,float16,fp8,0,4.345109303792317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,1,128,1,fp8,fp8,0,2.775760014851888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,2,128,1,float16,float16,0,3.7569281260172525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,2,128,1,fp8,fp8,0,2.7876160939534507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,2,128,1,float16,fp8,0,3.647226651509603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,4,128,1,float16,float16,0,3.7418079376220703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,4,128,1,float16,fp8,0,3.740367889404297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,4,128,1,fp8,fp8,0,2.8014132181803384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,8,128,1,float16,fp8,0,3.757685343424479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,8,128,1,float16,float16,0,4.048826535542806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,1,128,1,float16,float16,0,1.8895146052042644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,1,128,1,float16,fp8,0,1.8505919774373372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,40,8,128,1,fp8,fp8,0,2.839477221171061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,1,128,1,fp8,fp8,0,1.4141333897908528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,2,128,1,float16,float16,0,1.9560373624165852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,40,128,1,float16,float16,0,2.0368000666300454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,40,128,1,fp8,fp8,0,1.5398772557576497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,40,128,1,float16,fp8,0,1.9642400741577148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,2,128,1,float16,fp8,0,1.8527785936991374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,2,128,1,fp8,fp8,0,1.4187040328979492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,4,128,1,float16,float16,0,1.9039252599080403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,4,128,1,float16,fp8,0,1.8643892606099446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,4,128,1,fp8,fp8,0,1.4760853449503581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,8,128,1,float16,float16,0,1.9385120073954265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,8,128,1,float16,fp8,0,1.8729119300842285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,1,128,1,float16,float16,0,0.9891626834869385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,40,128,1,float16,float16,0,1.0320533116658528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,1,128,1,float16,fp8,0,0.9907999833424886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,40,8,128,1,fp8,fp8,0,1.443610668182373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,40,128,1,float16,fp8,0,1.0146400133768718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,1,128,1,fp8,fp8,0,0.7473653157552084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,40,128,1,fp8,fp8,0,0.8162773450215658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,2,128,1,float16,float16,0,0.9909706910451254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,2,128,1,float16,fp8,0,0.970970630645752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,2,128,1,fp8,fp8,0,0.7519626617431641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,4,128,1,float16,float16,0,1.0315093199412029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,4,128,1,float16,fp8,0,0.9738559722900391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,4,128,1,fp8,fp8,0,0.7557120323181152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,8,128,1,float16,fp8,0,0.9813386599222819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,8,128,1,float16,float16,0,1.0083946386973064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,1,128,1,float16,fp8,0,0.527786652247111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,1,128,1,float16,float16,0,0.5402719974517822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,40,8,128,1,fp8,fp8,0,0.765343983968099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,40,128,1,float16,float16,0,0.5538666645685831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,40,128,1,float16,fp8,0,0.5426986614863077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,40,128,1,fp8,fp8,0,0.47162667910257977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,1,128,1,fp8,fp8,0,0.4172373215357463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,2,128,1,float16,float16,0,0.5428266525268555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,2,128,1,float16,fp8,0,0.5288800001144409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,4,128,1,float16,fp8,0,0.5314773321151733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,2,128,1,fp8,fp8,0,0.41748801867167157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,4,128,1,fp8,fp8,0,0.41993598143259686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,4,128,1,float16,float16,0,0.5429439942042033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,8,128,1,float16,float16,0,0.565285325050354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,8,128,1,float16,fp8,0,0.5351893504460653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,1,128,1,float16,float16,0,0.3168800075848897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,40,8,128,1,fp8,fp8,0,0.42365864912668866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,40,128,1,float16,float16,0,0.3219040036201477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,1,128,1,float16,fp8,0,0.3078133265177409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,40,128,1,float16,fp8,0,0.31622399886449176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,1,128,1,fp8,fp8,0,0.2412053346633911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,2,128,1,float16,float16,0,0.3174026608467102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,40,128,1,fp8,fp8,0,0.255514661471049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,2,128,1,float16,fp8,0,0.3099199930826823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,2,128,1,fp8,fp8,0,0.23613866170247397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,4,128,1,float16,float16,0,0.31945600112279254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,4,128,1,fp8,fp8,0,0.23805866638819376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,4,128,1,float16,fp8,0,0.3104746739069621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,8,128,1,float16,float16,0,0.3290239969889323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,8,128,1,float16,fp8,0,0.31351999441782635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,40,8,128,1,fp8,fp8,0,0.24009066820144653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,1,128,1,float16,float16,0,8.103616078694662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,2,128,1,float16,float16,0,8.041648228963217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,1,128,1,fp8,fp8,0,6.1534989674886065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,2,128,1,float16,fp8,0,7.735029220581055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,2,128,1,fp8,fp8,0,6.788805643717448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,1,128,1,float16,fp8,0,7.962938944498698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,4,128,1,float16,float16,0,8.262079874674479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,4,128,1,float16,fp8,0,7.998421351114909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,1,128,1,float16,float16,0,3.9547252655029297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,4,128,1,fp8,fp8,0,6.226928075154622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,1,128,1,float16,fp8,0,3.859578768412272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,8,128,1,float16,float16,0,8.246357599894205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,8,128,1,fp8,fp8,0,6.274389266967773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,40,8,128,1,float16,fp8,0,7.914490381876628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,40,128,1,float16,float16,0,4.998469352722168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,40,128,1,fp8,fp8,0,3.3647680282592773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,40,128,1,float16,fp8,0,4.219733238220215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,1,128,1,fp8,fp8,0,3.043936093648275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,2,128,1,float16,fp8,0,3.853695869445801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,2,128,1,fp8,fp8,0,3.0438880920410156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,2,128,1,float16,float16,0,3.9478187561035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,4,128,1,float16,float16,0,3.9796320597330728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,4,128,1,float16,fp8,0,4.161301294962565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,4,128,1,fp8,fp8,0,3.0644960403442383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,8,128,1,float16,float16,0,4.059589385986328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,8,128,1,float16,fp8,0,3.9102452596028647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,1,128,1,float16,float16,0,2.0299253463745117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,1,128,1,float16,fp8,0,1.894986629486084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,1,128,1,fp8,fp8,0,1.5203893979390461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,2,128,1,float16,float16,0,1.9517439206441243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,40,8,128,1,fp8,fp8,0,3.107701301574707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,2,128,1,float16,fp8,0,1.981002648671468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,2,128,1,fp8,fp8,0,1.5307200749715169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,40,128,1,float16,float16,0,2.125903924306234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,40,128,1,fp8,fp8,0,1.7079253196716309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,40,128,1,float16,fp8,0,2.080549399058024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,4,128,1,float16,float16,0,1.9763466517130535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,4,128,1,float16,fp8,0,1.9105599721272786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,4,128,1,fp8,fp8,0,1.5402026176452637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,8,128,1,float16,fp8,0,1.9272799491882324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,8,128,1,float16,float16,0,2.109381357828776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,40,8,128,1,fp8,fp8,0,1.5630559921264648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,1,128,1,float16,float16,0,1.0055786768595378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,1,128,1,float16,fp8,0,0.9790026346842448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,40,128,1,float16,float16,0,1.090330680211385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,40,128,1,float16,fp8,0,1.0617813269297283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,1,128,1,fp8,fp8,0,0.7896906534830729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,2,128,1,float16,float16,0,1.0484639803568523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,40,128,1,fp8,fp8,0,0.8860586484273275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,2,128,1,fp8,fp8,0,0.7937599817911783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,2,128,1,float16,fp8,0,0.9806186358133951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,4,128,1,float16,float16,0,1.0138879617055256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,4,128,1,float16,fp8,0,0.9863946437835693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,4,128,1,fp8,fp8,0,0.7985173066457113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,8,128,1,float16,float16,0,1.0202879905700684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,1,128,1,float16,float16,0,0.5351253350575765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,8,128,1,float16,fp8,0,1.0472746690114338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,1,128,1,float16,fp8,0,0.5199360052744547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,40,8,128,1,fp8,fp8,0,0.808789332707723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,40,128,1,float16,float16,0,0.567413330078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,40,128,1,float16,fp8,0,0.5529386599858602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,40,128,1,fp8,fp8,0,0.4752853314081828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,1,128,1,fp8,fp8,0,0.4265066782633464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,2,128,1,float16,float16,0,0.53766401608785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,2,128,1,float16,fp8,0,0.5402346849441528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,2,128,1,fp8,fp8,0,0.42614932854970294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,4,128,1,float16,float16,0,0.5403733253479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,4,128,1,float16,fp8,0,0.5263199806213379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,8,128,1,float16,float16,0,0.5430399974187216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,4,128,1,fp8,fp8,0,0.42985065778096515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,8,128,1,fp8,fp8,0,0.4364000161488851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,1,128,1,float16,float16,0,0.30114134152730304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,40,8,128,1,float16,fp8,0,0.5299573342005411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,40,128,1,float16,float16,0,0.3511360088984172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,1,128,1,float16,fp8,0,0.29250667492548627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,40,128,1,fp8,fp8,0,0.26679466168085736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,40,128,1,float16,fp8,0,0.3065333366394043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,2,128,1,float16,float16,0,0.3015679915746053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,2,128,1,float16,fp8,0,0.2938399910926819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,2,128,1,fp8,fp8,0,0.2341066598892212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,1,128,1,fp8,fp8,0,0.23244265715281168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,4,128,1,float16,float16,0,0.31470932563145954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,4,128,1,float16,fp8,0,0.2953866720199585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,4,128,1,fp8,fp8,0,0.23477866252263388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,8,128,1,float16,fp8,0,0.297487994035085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,8,128,1,float16,float16,0,0.3053973317146301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,40,8,128,1,fp8,fp8,0,0.23775466283162436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,40,128,1,float16,float16,0,0.1781066656112671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,1,128,1,float16,float16,0,0.16844799121220908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,1,128,1,float16,fp8,0,0.16343999902407327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,40,128,1,fp8,fp8,0,0.16197333733240762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,1,128,1,fp8,fp8,0,0.13949867089589438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,40,128,1,float16,fp8,0,0.17471466461817423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,2,128,1,float16,float16,0,0.1681493322054545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,2,128,1,fp8,fp8,0,0.14089600245157877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,2,128,1,float16,fp8,0,0.16354667147000632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,4,128,1,float16,float16,0,0.16894400119781494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,4,128,1,float16,fp8,0,0.1641439994176229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,4,128,1,fp8,fp8,0,0.14474667112032572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,8,128,1,float16,float16,0,0.1697439948717753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,8,128,1,fp8,fp8,0,0.14333333571751913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,40,8,128,1,float16,fp8,0,0.16530133287111917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,1,128,1,float16,fp8,0,4.923349380493164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,1,128,1,float16,float16,0,5.18507194519043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,1,128,1,fp8,fp8,0,4.148058573404948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,2,128,1,float16,float16,0,5.269674619038899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,2,128,1,float16,fp8,0,5.433493296305339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,2,128,1,fp8,fp8,0,4.156074523925781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,4,128,1,float16,float16,0,5.243018786112468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,4,128,1,float16,fp8,0,4.973861376444499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,1,128,1,float16,float16,0,2.546074708302816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,4,128,1,fp8,fp8,0,4.202783902486165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,8,128,1,fp8,fp8,0,4.205909411112468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,8,128,1,float16,float16,0,5.3038028081258135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,40,8,128,1,float16,fp8,0,5.029221216837565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,40,128,1,float16,float16,0,3.383962631225586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,1,128,1,float16,fp8,0,2.4337919553120932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,40,128,1,float16,fp8,0,2.72705078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,40,128,1,fp8,fp8,0,2.3147199948628745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,1,128,1,fp8,fp8,0,2.0536319414774575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,2,128,1,float16,float16,0,2.5557920138041177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,2,128,1,float16,fp8,0,2.4453333218892417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,2,128,1,fp8,fp8,0,2.053808053334554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,4,128,1,float16,float16,0,2.8184054692586265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,4,128,1,fp8,fp8,0,2.0697546005249023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,4,128,1,float16,fp8,0,2.4527573585510254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,8,128,1,float16,float16,0,2.602549393971761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,8,128,1,float16,fp8,0,2.4903945922851562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,1,128,1,float16,float16,0,1.2692906856536865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,1,128,1,float16,fp8,0,1.2295520305633545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,1,128,1,fp8,fp8,0,1.0290186405181885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,40,8,128,1,fp8,fp8,0,2.09822940826416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,2,128,1,float16,float16,0,1.2723946571350098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,40,128,1,float16,float16,0,1.4214612642923992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,40,128,1,float16,fp8,0,1.3816426595052083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,40,128,1,fp8,fp8,0,1.3605012893676758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,2,128,1,float16,fp8,0,1.2346080144246419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,2,128,1,fp8,fp8,0,1.0331466992696126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,4,128,1,float16,float16,0,1.2793973286946614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,4,128,1,float16,fp8,0,1.2451573212941487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,4,128,1,fp8,fp8,0,1.0480426947275798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,8,128,1,float16,fp8,0,1.258074680964152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,8,128,1,float16,float16,0,1.2915093104044597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,1,128,1,float16,float16,0,0.6621973514556885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,1,128,1,float16,fp8,0,0.6416053374608358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,40,8,128,1,fp8,fp8,0,1.1113920211791992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,40,128,1,float16,float16,0,0.7273226579030355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,1,128,1,fp8,fp8,0,0.5400960048039755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,2,128,1,float16,fp8,0,0.6440266768137614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,2,128,1,float16,float16,0,0.6638186772664388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,40,128,1,fp8,fp8,0,0.6151146491368612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,2,128,1,fp8,fp8,0,0.545194665590922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,4,128,1,float16,float16,0,0.701802651087443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,40,128,1,float16,fp8,0,0.7036853631337484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,4,128,1,float16,fp8,0,0.6476373275121053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,4,128,1,fp8,fp8,0,0.5468586683273315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,8,128,1,float16,fp8,0,0.6543893416722616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,8,128,1,float16,float16,0,0.6730826695760092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,1,128,1,float16,float16,0,0.35655466715494794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,40,8,128,1,fp8,fp8,0,0.5549333492914835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,40,128,1,float16,float16,0,0.3852693239847819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,1,128,1,float16,fp8,0,0.34699734052022296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,40,128,1,float16,fp8,0,0.4331413507461548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,40,128,1,fp8,fp8,0,0.3343840042750041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,1,128,1,fp8,fp8,0,0.2970293362935384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,2,128,1,float16,float16,0,0.35903998215993244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,2,128,1,float16,fp8,0,0.34781332810719806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,2,128,1,fp8,fp8,0,0.29729066292444867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,4,128,1,float16,fp8,0,0.35040533542633057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,4,128,1,float16,float16,0,0.36101865768432617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,4,128,1,fp8,fp8,0,0.3098026712735494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,1,128,1,float16,float16,0,0.20317333936691284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,8,128,1,float16,fp8,0,0.35415999094645184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,1,128,1,float16,fp8,0,0.20106132825215658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,8,128,1,float16,float16,0,0.36267733573913574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,40,8,128,1,fp8,fp8,0,0.3028159936269124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,40,128,1,float16,float16,0,0.21633066733678183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,40,128,1,float16,fp8,0,0.2116853396097819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,40,128,1,fp8,fp8,0,0.18467199802398682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,1,128,1,fp8,fp8,0,0.1637440025806427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,2,128,1,float16,float16,0,0.20427733659744263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,2,128,1,float16,fp8,0,0.20281066497166952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,4,128,1,float16,float16,0,0.20614933967590332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,4,128,1,float16,fp8,0,0.1996906598409017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,4,128,1,fp8,fp8,0,0.16500266393025717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,2,128,1,fp8,fp8,0,0.16400532921155295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,8,128,1,float16,float16,0,0.20803199211756387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,8,128,1,fp8,fp8,0,0.16841065883636475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,40,8,128,1,float16,fp8,0,0.20257065693537393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,40,128,1,float16,float16,0,0.1397226651509603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,1,128,1,float16,float16,0,0.116047998269399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,40,128,1,float16,fp8,0,0.12219732999801636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,40,128,1,fp8,fp8,0,0.11191466450691223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,1,128,1,float16,fp8,0,0.11289599537849426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,2,128,1,float16,float16,0,0.11680533488591512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,2,128,1,float16,fp8,0,0.114138662815094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,2,128,1,fp8,fp8,0,0.10152000188827515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,4,128,1,float16,float16,0,0.11636267105738322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,4,128,1,float16,fp8,0,0.11500799655914307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,4,128,1,fp8,fp8,0,0.10211732983589172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,1,128,1,fp8,fp8,0,0.09945066769917806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,8,128,1,float16,fp8,0,0.11495467027028401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,8,128,1,fp8,fp8,0,0.10437333583831787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,40,8,128,1,float16,float16,0,0.11820266644159953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,1,128,1,float16,float16,0,5.232405344645183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,1,128,1,fp8,fp8,0,4.029557228088379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,1,128,1,float16,fp8,0,4.735589345296224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,2,128,1,float16,float16,0,4.844170570373535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,2,128,1,fp8,fp8,0,4.0774186452229815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,4,128,1,float16,float16,0,5.000570615132649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,4,128,1,float16,fp8,0,5.005130767822266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,2,128,1,float16,fp8,0,4.924208005269368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,1,128,1,float16,float16,0,2.385690689086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,4,128,1,fp8,fp8,0,4.465471903483073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,8,128,1,float16,float16,0,5.2643998463948565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,8,128,1,fp8,fp8,0,4.235376040140788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,40,8,128,1,float16,fp8,0,5.145008087158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,40,128,1,float16,float16,0,3.1533333460489907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,40,128,1,float16,fp8,0,2.9982614517211914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,1,128,1,fp8,fp8,0,2.0488533973693848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,1,128,1,float16,fp8,0,2.3832480112711587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,40,128,1,fp8,fp8,0,2.527008056640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,2,128,1,float16,float16,0,2.4023733139038086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,2,128,1,fp8,fp8,0,2.0220160484313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,2,128,1,float16,fp8,0,2.3976853688557944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,4,128,1,float16,float16,0,2.4281439781188965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,4,128,1,float16,fp8,0,2.49233071009318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,4,128,1,fp8,fp8,0,2.236255963643392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,8,128,1,float16,float16,0,2.572277386983236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,1,128,1,float16,float16,0,1.204591989517212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,8,128,1,float16,fp8,0,2.5545493761698403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,1,128,1,float16,fp8,0,1.3272159894307454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,1,128,1,fp8,fp8,0,0.9788533051808676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,40,8,128,1,fp8,fp8,0,2.117173353830973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,2,128,1,float16,float16,0,1.2123093605041504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,40,128,1,float16,float16,0,1.571946620941162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,2,128,1,float16,fp8,0,1.2114453315734863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,40,128,1,float16,fp8,0,1.4909173647562664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,2,128,1,fp8,fp8,0,1.0495039621988933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,40,128,1,fp8,fp8,0,1.2643413543701172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,4,128,1,float16,float16,0,1.2291733423868816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,4,128,1,float16,fp8,0,1.2289439837137859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,4,128,1,fp8,fp8,0,1.0474879741668701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,1,128,1,float16,float16,0,0.6166613499323527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,8,128,1,float16,float16,0,1.255717356999715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,8,128,1,fp8,fp8,0,1.0545066992441814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,40,8,128,1,float16,fp8,0,1.2663040161132812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,40,128,1,float16,float16,0,1.036234696706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,1,128,1,float16,fp8,0,0.6156853437423706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,40,128,1,float16,fp8,0,0.7477066516876221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,1,128,1,fp8,fp8,0,0.5012853145599365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,2,128,1,float16,float16,0,0.6199146509170532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,2,128,1,float16,fp8,0,0.6195786794026693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,40,128,1,fp8,fp8,0,0.637392004330953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,2,128,1,fp8,fp8,0,0.5063039859135946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,4,128,1,float16,float16,0,0.7047200202941895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,4,128,1,fp8,fp8,0,0.5151786804199219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,4,128,1,float16,fp8,0,0.6256426572799683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,8,128,1,float16,float16,0,0.6422826846440634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,8,128,1,float16,fp8,0,0.6388426621754965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,40,8,128,1,fp8,fp8,0,0.5340906778971354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,1,128,1,float16,float16,0,0.31918400526046753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,1,128,1,float16,fp8,0,0.3186239997545878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,40,128,1,float16,float16,0,0.40563201904296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,1,128,1,fp8,fp8,0,0.26109333833058673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,40,128,1,float16,fp8,0,0.38710931936899823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,2,128,1,float16,float16,0,0.3203253348668416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,40,128,1,fp8,fp8,0,0.4047146638234456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,2,128,1,float16,fp8,0,0.3205920060475667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,2,128,1,fp8,fp8,0,0.2637439966201782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,4,128,1,float16,fp8,0,0.3259626626968384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,4,128,1,float16,float16,0,0.32500267028808594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,4,128,1,fp8,fp8,0,0.2670560081799825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,1,128,1,float16,float16,0,0.1722453236579895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,8,128,1,float16,float16,0,0.3321546713511149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,8,128,1,float16,fp8,0,0.3305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,40,8,128,1,fp8,fp8,0,0.29755733410517377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,40,128,1,float16,float16,0,0.21633599201838175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,40,128,1,fp8,fp8,0,0.17528533935546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,1,128,1,float16,fp8,0,0.17302932341893515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,1,128,1,fp8,fp8,0,0.1332373321056366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,2,128,1,float16,float16,0,0.17427732547124228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,40,128,1,float16,fp8,0,0.20796799659729004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,2,128,1,float16,fp8,0,0.17463467518488565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,2,128,1,fp8,fp8,0,0.14332266648610434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,4,128,1,fp8,fp8,0,0.13664000233014426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,4,128,1,float16,float16,0,0.17690666516621908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,4,128,1,float16,fp8,0,0.17787200212478638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,8,128,1,float16,fp8,0,0.1785866618156433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,8,128,1,float16,float16,0,0.18021865685780844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,40,128,1,float16,float16,0,0.12309333682060242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,40,8,128,1,fp8,fp8,0,0.13948266704877219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,40,128,1,float16,fp8,0,0.15940800309181213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,1,128,1,float16,float16,0,0.0941439966360728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,1,128,1,float16,fp8,0,0.09386133154233296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,40,128,1,fp8,fp8,0,0.0941546658674876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,1,128,1,fp8,fp8,0,0.07494933406511943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,2,128,1,float16,fp8,0,0.0944160024325053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,2,128,1,fp8,fp8,0,0.07599466542402904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,4,128,1,float16,float16,0,0.09857066472371419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,2,128,1,float16,float16,0,0.09419199824333191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,4,128,1,float16,fp8,0,0.09532800316810608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,4,128,1,fp8,fp8,0,0.0777706652879715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,8,128,1,float16,float16,0,0.09636267026265462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,8,128,1,fp8,fp8,0,0.07982400059700012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,40,8,128,1,float16,fp8,0,0.09609599908192952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,40,128,1,float16,float16,0,0.0641546646753947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,40,128,1,float16,fp8,0,0.0691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,1,128,1,float16,float16,0,0.05283733208974203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,1,128,1,float16,fp8,0,0.0533493310213089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,1,128,1,fp8,fp8,0,0.043706665436426796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,2,128,1,float16,float16,0,0.052933335304260254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,2,128,1,float16,fp8,0,0.05352533360322317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,4,128,1,float16,float16,0,0.053344001372655235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,2,128,1,fp8,fp8,0,0.04430399835109711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,40,128,1,fp8,fp8,0,0.05499200026194254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,4,128,1,fp8,fp8,0,0.04613333443800608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,4,128,1,float16,fp8,0,0.05409066875775655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,8,128,1,float16,float16,0,0.05433600147565206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,8,128,1,float16,fp8,0,0.05505066613356272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,40,8,128,1,fp8,fp8,0,0.047744000951449074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,1,128,1,float16,fp8,0,3.704517364501953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,1,128,1,float16,float16,0,4.20796267191569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,2,128,1,fp8,fp8,0,3.389637311299642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,2,128,1,float16,float16,0,3.7374134063720703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,4,128,1,float16,float16,0,3.9565439224243164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,1,128,1,fp8,fp8,0,3.3970346450805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,4,128,1,float16,fp8,0,3.9502293268839517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,2,128,1,float16,fp8,0,3.752314567565918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,1,128,1,float16,float16,0,1.865455945332845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,4,128,1,fp8,fp8,0,3.4735466639200845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,8,128,1,float16,float16,0,4.978906631469727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,8,128,1,fp8,fp8,0,3.5642293294270835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,1,128,1,float16,fp8,0,1.8641759554545085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,40,8,128,1,float16,fp8,0,4.1221920649210615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,1,128,1,fp8,fp8,0,1.653696060180664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,40,128,1,float16,float16,0,2.662602742513021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,40,128,1,fp8,fp8,0,2.190122604370117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,2,128,1,float16,float16,0,2.1682559649149575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,2,128,1,float16,fp8,0,1.87936004002889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,2,128,1,fp8,fp8,0,1.6856853167215984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,40,128,1,float16,fp8,0,2.534122625986735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,4,128,1,float16,float16,0,1.9085440635681152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,4,128,1,float16,fp8,0,1.9770719210306804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,4,128,1,fp8,fp8,0,1.7336640357971191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,8,128,1,float16,float16,0,2.051413377126058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,1,128,1,float16,float16,0,0.9448959827423096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,8,128,1,float16,fp8,0,2.4318933486938477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,1,128,1,float16,fp8,0,0.9448213577270508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,40,8,128,1,fp8,fp8,0,1.777488072713216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,1,128,1,fp8,fp8,0,0.8091359933217367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,40,128,1,float16,float16,0,1.3347093264261882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,40,128,1,fp8,fp8,0,1.1044853528340657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,2,128,1,float16,fp8,0,1.0915839672088623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,2,128,1,float16,float16,0,0.9545439879099528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,40,128,1,float16,fp8,0,1.2452106475830078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,2,128,1,fp8,fp8,0,0.8452426592508951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,4,128,1,float16,float16,0,0.967583974202474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,4,128,1,float16,fp8,0,0.9659679730733236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,4,128,1,fp8,fp8,0,0.8646186987559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,8,128,1,float16,float16,0,1.002234697341919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,8,128,1,fp8,fp8,0,0.8872693379720052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,40,8,128,1,float16,fp8,0,1.1882346471150715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,40,128,1,float16,fp8,0,0.6281013488769531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,40,128,1,float16,float16,0,0.6677172978719076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,1,128,1,float16,fp8,0,0.48229865233103436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,1,128,1,float16,float16,0,0.4824426571528117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,1,128,1,fp8,fp8,0,0.41522665818532306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,2,128,1,float16,float16,0,0.4875893195470174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,2,128,1,float16,fp8,0,0.5555893182754517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,40,128,1,fp8,fp8,0,0.5540693203608195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,2,128,1,fp8,fp8,0,0.4189866781234741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,4,128,1,float16,float16,0,0.49515199661254883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,4,128,1,float16,fp8,0,0.49350400765736896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,4,128,1,fp8,fp8,0,0.4313439925511678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,8,128,1,float16,float16,0,0.5088320175806681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,8,128,1,float16,fp8,0,0.5075039863586426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,1,128,1,float16,float16,0,0.24885867039362589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,1,128,1,fp8,fp8,0,0.21645333369572958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,40,8,128,1,fp8,fp8,0,0.487829327583313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,1,128,1,float16,fp8,0,0.24827200174331665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,2,128,1,float16,float16,0,0.25166932741800946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,40,128,1,float16,float16,0,0.3437653382619222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,40,128,1,float16,fp8,0,0.3255573312441508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,40,128,1,fp8,fp8,0,0.28257066011428833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,2,128,1,float16,fp8,0,0.251093327999115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,2,128,1,fp8,fp8,0,0.2184106707572937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,4,128,1,float16,float16,0,0.29339200258255005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,4,128,1,fp8,fp8,0,0.22281599044799805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,4,128,1,float16,fp8,0,0.2565386692682902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,8,128,1,float16,float16,0,0.26394667228062946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,8,128,1,float16,fp8,0,0.26174400250116986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,1,128,1,float16,float16,0,0.1341866652170817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,40,8,128,1,fp8,fp8,0,0.22874667247136435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,1,128,1,float16,fp8,0,0.13467199603716531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,1,128,1,fp8,fp8,0,0.1116480032602946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,40,128,1,float16,float16,0,0.18219733238220215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,40,128,1,float16,fp8,0,0.17408533891042074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,2,128,1,float16,float16,0,0.13578133781750998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,40,128,1,fp8,fp8,0,0.21910399198532104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,2,128,1,float16,fp8,0,0.13556800285975137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,4,128,1,float16,fp8,0,0.13942933082580566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,4,128,1,fp8,fp8,0,0.11507200201352437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,8,128,1,float16,float16,0,0.1413226624329885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,8,128,1,float16,fp8,0,0.14112533132235208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,4,128,1,float16,float16,0,0.13883733749389648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,2,128,1,fp8,fp8,0,0.1127839982509613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,40,128,1,float16,float16,0,0.11173866192499797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,40,8,128,1,fp8,fp8,0,0.13250666856765747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,1,128,1,float16,float16,0,0.0774239997069041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,1,128,1,float16,fp8,0,0.07789333164691925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,1,128,1,fp8,fp8,0,0.06363200147946675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,40,128,1,float16,fp8,0,0.10703999797503154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,40,128,1,fp8,fp8,0,0.08353066444396973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,2,128,1,float16,float16,0,0.0777759999036789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,2,128,1,float16,fp8,0,0.07814933359622955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,2,128,1,fp8,fp8,0,0.06857066849867503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,4,128,1,float16,float16,0,0.07814933359622955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,4,128,1,float16,fp8,0,0.07868800063927968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,4,128,1,fp8,fp8,0,0.06683200101057689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,8,128,1,float16,fp8,0,0.08061333497365315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,8,128,1,float16,float16,0,0.08057066798210144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,40,128,1,float16,float16,0,0.05596800148487091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,40,8,128,1,fp8,fp8,0,0.06850666801134746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,1,128,1,float16,float16,0,0.04465599854787191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,40,128,1,float16,fp8,0,0.060218666990598045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,40,128,1,fp8,fp8,0,0.04882133503754934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,1,128,1,fp8,fp8,0,0.03810133288304011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,2,128,1,float16,fp8,0,0.045007998744646706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,1,128,1,float16,fp8,0,0.04505600035190582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,4,128,1,float16,float16,0,0.044778664906819664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,4,128,1,float16,fp8,0,0.04560533165931702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,4,128,1,fp8,fp8,0,0.040720000863075256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,2,128,1,float16,float16,0,0.0444106658299764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,8,128,1,float16,float16,0,0.04613333443800608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,2,128,1,fp8,fp8,0,0.03824000060558319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,8,128,1,float16,fp8,0,0.04682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,40,128,1,float16,float16,0,0.0330826664964358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,40,128,1,float16,fp8,0,0.031680000325044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,40,8,128,1,fp8,fp8,0,0.0421013335386912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,40,128,1,fp8,fp8,0,0.029103999336560566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,1,128,1,float16,fp8,0,0.029493334392706554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,1,128,1,float16,float16,0,0.029103999336560566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,2,128,1,float16,float16,0,0.02959466725587845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,2,128,1,float16,fp8,0,0.029391999046007793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,1,128,1,fp8,fp8,0,0.026373334228992462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,4,128,1,float16,fp8,0,0.029333333174387615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,4,128,1,float16,float16,0,0.029472000896930695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,4,128,1,fp8,fp8,0,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,2,128,1,fp8,fp8,0,0.026682667434215546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,8,128,1,float16,float16,0,0.030037333567937214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,8,128,1,float16,fp8,0,0.03032533327738444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,40,8,128,1,fp8,fp8,0,0.028864001234372456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,1,128,1,float16,fp8,0,1.6155519485473633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,1,128,1,fp8,fp8,0,1.31987198193868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,2,128,1,float16,fp8,0,1.9386879603068035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,2,128,1,fp8,fp8,0,1.3942933082580566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,4,128,1,float16,float16,0,1.6640480359395344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,4,128,1,float16,fp8,0,1.7231359481811523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,1,128,1,float16,float16,0,1.6194559733072917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,2,128,1,float16,float16,0,1.6346880594889324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,4,128,1,fp8,fp8,0,1.4564426740010579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,1,128,1,float16,float16,0,0.8215839862823486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,8,128,1,float16,float16,0,1.7962506612141926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,8,128,1,fp8,fp8,0,1.5368746121724446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,40,128,1,float16,float16,0,1.2380800247192383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,40,128,1,float16,fp8,0,1.1651466687520344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,40,8,128,1,float16,fp8,0,2.1797760327657065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,1,128,1,float16,fp8,0,0.8198346296946207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,40,128,1,fp8,fp8,0,0.9727146625518799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,1,128,1,fp8,fp8,0,0.6631093422571818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,2,128,1,float16,float16,0,0.827893336613973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,2,128,1,float16,fp8,0,0.8281813462575277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,2,128,1,fp8,fp8,0,0.7001173496246338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,4,128,1,float16,float16,0,0.8424853483835856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,4,128,1,float16,fp8,0,1.0193599859873455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,4,128,1,fp8,fp8,0,0.7291946411132812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,8,128,1,float16,float16,0,0.8710347016652426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,8,128,1,float16,fp8,0,0.8665973345438639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,1,128,1,float16,fp8,0,0.4184906482696533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,1,128,1,float16,float16,0,0.48670931657155353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,40,8,128,1,fp8,fp8,0,0.7654346625010172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,40,128,1,float16,float16,0,0.6222986777623495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,40,128,1,float16,fp8,0,0.5785333315531412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,40,128,1,fp8,fp8,0,0.48972801367441815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,1,128,1,fp8,fp8,0,0.3409493366877238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,2,128,1,float16,float16,0,0.42417601744333905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,2,128,1,float16,fp8,0,0.42282132307688397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,2,128,1,fp8,fp8,0,0.39562666416168213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,4,128,1,float16,fp8,0,0.4298559824625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,4,128,1,fp8,fp8,0,0.3643733263015747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,4,128,1,float16,float16,0,0.43164265155792236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,8,128,1,float16,float16,0,0.4465013345082601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,8,128,1,fp8,fp8,0,0.3858026663462321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,40,128,1,float16,fp8,0,0.46004267533620197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,40,128,1,float16,float16,0,0.3190400004386902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,40,8,128,1,float16,fp8,0,0.4423786799112956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,1,128,1,float16,float16,0,0.21609600385030112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,1,128,1,float16,fp8,0,0.21649599075317383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,1,128,1,fp8,fp8,0,0.1808746655782064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,2,128,1,float16,float16,0,0.21811733643213907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,40,128,1,fp8,fp8,0,0.25082133213679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,2,128,1,fp8,fp8,0,0.18178667624791464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,2,128,1,float16,fp8,0,0.24977600574493408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,4,128,1,float16,fp8,0,0.2237066626548767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,4,128,1,float16,float16,0,0.22277865807215372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,4,128,1,fp8,fp8,0,0.18978132804234824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,8,128,1,float16,float16,0,0.22959466775258383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,8,128,1,float16,fp8,0,0.22849067052205405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,1,128,1,float16,float16,0,0.11759466926256816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,40,8,128,1,fp8,fp8,0,0.19708265860875449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,40,128,1,float16,float16,0,0.16967999935150146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,40,128,1,float16,fp8,0,0.23875733216603598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,40,128,1,fp8,fp8,0,0.14315733313560486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,1,128,1,float16,fp8,0,0.11703999837239583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,1,128,1,fp8,fp8,0,0.10079466303189595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,2,128,1,float16,float16,0,0.11858666936556499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,2,128,1,fp8,fp8,0,0.10128532846768697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,2,128,1,float16,fp8,0,0.11987200379371643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,4,128,1,float16,float16,0,0.1213653286298116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,4,128,1,float16,fp8,0,0.13983999689420065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,4,128,1,fp8,fp8,0,0.10382933417956035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,8,128,1,float16,fp8,0,0.12392000357309978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,8,128,1,float16,float16,0,0.1249120036760966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,40,8,128,1,fp8,fp8,0,0.10587199529012044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,40,128,1,float16,fp8,0,0.10078932841618855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,1,128,1,float16,float16,0,0.07025066514809926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,40,128,1,float16,float16,0,0.1043893297513326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,40,128,1,fp8,fp8,0,0.0788800021012624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,1,128,1,float16,fp8,0,0.06962133447329204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,1,128,1,fp8,fp8,0,0.057578667998313904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,2,128,1,float16,float16,0,0.0690773328145345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,2,128,1,float16,fp8,0,0.06994133194287618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,2,128,1,fp8,fp8,0,0.05795200169086456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,4,128,1,float16,fp8,0,0.07069333394368489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,4,128,1,fp8,fp8,0,0.05951466659704844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,4,128,1,float16,float16,0,0.07430399954319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,8,128,1,float16,float16,0,0.07277333239714305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,8,128,1,float16,fp8,0,0.07251200079917908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,40,128,1,float16,float16,0,0.052341332038243614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,40,8,128,1,fp8,fp8,0,0.061530664563179016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,1,128,1,float16,float16,0,0.04102933406829834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,40,128,1,float16,fp8,0,0.05043200155099233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,1,128,1,float16,fp8,0,0.04115733255942663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,40,128,1,fp8,fp8,0,0.046480000019073486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,2,128,1,float16,float16,0,0.041162667175134025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,2,128,1,float16,fp8,0,0.04126933217048645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,1,128,1,fp8,fp8,0,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,2,128,1,fp8,fp8,0,0.035749333600203194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,4,128,1,float16,float16,0,0.04163199911514918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,4,128,1,float16,fp8,0,0.04214400053024292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,8,128,1,float16,float16,0,0.0420959989229838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,4,128,1,fp8,fp8,0,0.03724266588687897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,8,128,1,float16,fp8,0,0.04318400224049886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,40,8,128,1,fp8,fp8,0,0.03882666677236557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,1,128,1,float16,float16,0,0.028431999186674755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,1,128,1,float16,fp8,0,0.02846933404604594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,40,128,1,float16,fp8,0,0.03141333411137263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,40,128,1,fp8,fp8,0,0.029674666623274486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,1,128,1,fp8,fp8,0,0.02646933247645696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,40,128,1,float16,float16,0,0.031061333914597828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,2,128,1,float16,float16,0,0.02850666642189026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,2,128,1,float16,fp8,0,0.028565332293510437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,2,128,1,fp8,fp8,0,0.026186667382717133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,4,128,1,float16,float16,0,0.02887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,8,128,1,float16,float16,0,0.029045333464940388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,4,128,1,fp8,fp8,0,0.026890667776266735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,4,128,1,float16,fp8,0,0.02868266652027766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,8,128,1,float16,fp8,0,0.03881066789229711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,40,8,128,1,fp8,fp8,0,0.028954667349656422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,40,128,1,float16,float16,0,0.022277332842350006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,40,128,1,fp8,fp8,0,0.021146667500336964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,1,128,1,float16,float16,0,0.021530665457248688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,40,128,1,float16,fp8,0,0.022800001005331676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,1,128,1,float16,fp8,0,0.021477334201335907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,1,128,1,fp8,fp8,0,0.01998399943113327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,2,128,1,float16,fp8,0,0.021344001094500225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,2,128,1,fp8,fp8,0,0.01993600030740102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,2,128,1,float16,float16,0,0.02046400060256322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,4,128,1,float16,float16,0,0.020954666038354237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,4,128,1,float16,fp8,0,0.021674667795499165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,8,128,1,float16,float16,0,0.021045332153638203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,4,128,1,fp8,fp8,0,0.020224000016848247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,8,128,1,fp8,fp8,0,0.021317332983016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,40,8,128,1,float16,fp8,0,0.021850667893886566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,1,128,1,fp8,fp8,0,0.661242683728536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,1,128,1,float16,float16,0,0.7743039925893148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,2,128,1,float16,float16,0,0.9671200116475424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,2,128,1,fp8,fp8,0,0.7066400051116943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,1,128,1,float16,fp8,0,0.7736159960428873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,2,128,1,float16,fp8,0,0.777562697728475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,4,128,1,float16,float16,0,0.7944426536560059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,4,128,1,float16,fp8,0,0.7911093235015869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,1,128,1,float16,float16,0,0.39769065380096436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,4,128,1,fp8,fp8,0,0.7347359657287598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,8,128,1,float16,float16,0,0.8206506570180258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,40,128,1,fp8,fp8,0,0.4889920155207316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,8,128,1,float16,fp8,0,1.079973300298055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,40,8,128,1,fp8,fp8,0,0.7685973644256592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,40,128,1,float16,fp8,0,0.6049600044886271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,40,128,1,float16,float16,0,0.654416004816691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,1,128,1,float16,fp8,0,0.39707199732462567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,1,128,1,fp8,fp8,0,0.3420693476994832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,2,128,1,float16,float16,0,0.40164800484975177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,2,128,1,float16,fp8,0,0.40114665031433105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,2,128,1,fp8,fp8,0,0.41022932529449463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,4,128,1,float16,float16,0,0.40938134988149005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,4,128,1,float16,fp8,0,0.4078720013300578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,4,128,1,fp8,fp8,0,0.3651253382364909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,1,128,1,float16,float16,0,0.2055786649386088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,8,128,1,float16,float16,0,0.4257066647211711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,8,128,1,float16,fp8,0,0.42126933733622235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,1,128,1,float16,fp8,0,0.20641066630681357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,40,8,128,1,fp8,fp8,0,0.3874080181121826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,40,128,1,float16,fp8,0,0.46214401721954346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,40,128,1,float16,float16,0,0.3333280086517334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,40,128,1,fp8,fp8,0,0.25049599011739093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,1,128,1,fp8,fp8,0,0.17986667156219482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,2,128,1,float16,float16,0,0.20811732610066733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,2,128,1,fp8,fp8,0,0.18336532513300577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,2,128,1,float16,fp8,0,0.20801067352294922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,4,128,1,float16,float16,0,0.21339199940363565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,4,128,1,float16,fp8,0,0.2565760016441345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,4,128,1,fp8,fp8,0,0.1896053353945414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,8,128,1,float16,fp8,0,0.21808000405629477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,8,128,1,float16,float16,0,0.2208426594734192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,40,8,128,1,fp8,fp8,0,0.19709867238998413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,40,128,1,float16,fp8,0,0.1678239901860555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,1,128,1,float16,float16,0,0.12412266929944356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,40,128,1,float16,float16,0,0.1783626675605774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,1,128,1,float16,fp8,0,0.11351466178894043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,40,128,1,fp8,fp8,0,0.1362986663977305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,1,128,1,fp8,fp8,0,0.0993173321088155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,2,128,1,float16,float16,0,0.11365333199501038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,2,128,1,float16,fp8,0,0.11416000127792358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,2,128,1,fp8,fp8,0,0.10084799925486247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,4,128,1,fp8,fp8,0,0.10337066650390625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,4,128,1,float16,float16,0,0.1368160049120585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,8,128,1,float16,float16,0,0.12012799580891927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,8,128,1,float16,fp8,0,0.11926399668057759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,8,128,1,fp8,fp8,0,0.10620799660682678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,40,4,128,1,float16,fp8,0,0.1181706686814626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,40,128,1,float16,float16,0,0.10029333829879761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,1,128,1,fp8,fp8,0,0.05671466886997223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,1,128,1,float16,fp8,0,0.06570666531721751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,2,128,1,float16,float16,0,0.06625066697597504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,40,128,1,fp8,fp8,0,0.07891199986139934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,40,128,1,float16,fp8,0,0.09563199679056804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,1,128,1,float16,float16,0,0.0674239993095398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,2,128,1,fp8,fp8,0,0.057855998476346336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,4,128,1,float16,float16,0,0.06709333260854085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,4,128,1,float16,fp8,0,0.06764266888300578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,2,128,1,float16,fp8,0,0.06669333577156067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,4,128,1,fp8,fp8,0,0.05938666562239329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,8,128,1,float16,float16,0,0.07027733325958252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,8,128,1,float16,fp8,0,0.06977066894372304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,40,8,128,1,fp8,fp8,0,0.06947733461856842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,40,128,1,float16,float16,0,0.05100800096988678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,40,128,1,float16,fp8,0,0.04853333532810211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,1,128,1,float16,fp8,0,0.03937066594759623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,1,128,1,fp8,fp8,0,0.03527999917666117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,40,128,1,fp8,fp8,0,0.04571199913819631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,2,128,1,float16,float16,0,0.03935466706752777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,1,128,1,float16,float16,0,0.0395359992980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,2,128,1,float16,fp8,0,0.039861333866914116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,2,128,1,fp8,fp8,0,0.035445332527160645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,4,128,1,float16,float16,0,0.039701332648595176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,4,128,1,fp8,fp8,0,0.03760000069936117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,4,128,1,float16,fp8,0,0.04043733328580856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,8,128,1,float16,float16,0,0.04092800120512644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,8,128,1,float16,fp8,0,0.041834667325019836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,40,8,128,1,fp8,fp8,0,0.03929600119590759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,40,128,1,float16,float16,0,0.03138133386770884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,1,128,1,float16,float16,0,0.028501334289709728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,40,128,1,float16,fp8,0,0.030970667799313862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,40,128,1,fp8,fp8,0,0.02900800108909607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,1,128,1,float16,fp8,0,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,1,128,1,fp8,fp8,0,0.026154667139053345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,2,128,1,float16,float16,0,0.0284853329261144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,2,128,1,fp8,fp8,0,0.02609066665172577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,4,128,1,float16,float16,0,0.028789333999156952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,8,128,1,float16,fp8,0,0.029738667110602062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,2,128,1,float16,fp8,0,0.02870933214823405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,8,128,1,fp8,fp8,0,0.028373333315054577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,4,128,1,fp8,fp8,0,0.027285332481066387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,4,128,1,float16,fp8,0,0.02906133234500885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,40,8,128,1,float16,float16,0,0.029152000943819683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,1,128,1,float16,fp8,0,0.020224000016848247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,40,128,1,float16,fp8,0,0.02183466653029124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,40,128,1,float16,float16,0,0.021776000658671062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,40,128,1,fp8,fp8,0,0.021946666141351063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,1,128,1,fp8,fp8,0,0.01974933346112569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,1,128,1,float16,float16,0,0.019744000087181728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,2,128,1,float16,float16,0,0.020256000260512035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,2,128,1,fp8,fp8,0,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,2,128,1,float16,fp8,0,0.020703999946514767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,4,128,1,float16,float16,0,0.020495999604463577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,4,128,1,float16,fp8,0,0.02072000006834666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,8,128,1,float16,float16,0,0.02045866722861926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,4,128,1,fp8,fp8,0,0.020367999871571858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,8,128,1,fp8,fp8,0,0.020634666085243225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,40,128,1,float16,float16,0,0.017877332866191864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,1,128,1,float16,fp8,0,0.017237332959969837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,1,128,1,float16,float16,0,0.017349333812793095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,40,128,1,fp8,fp8,0,0.018426666657129925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,40,8,128,1,float16,fp8,0,0.020410666863123577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,40,128,1,float16,fp8,0,0.018506667266289394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,1,128,1,fp8,fp8,0,0.017786666750907898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,2,128,1,float16,float16,0,0.017312000195185345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,2,128,1,float16,fp8,0,0.017605333278576534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,2,128,1,fp8,fp8,0,0.01752000053723653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,4,128,1,float16,fp8,0,0.017898666361967724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,4,128,1,float16,float16,0,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,8,128,1,float16,fp8,0,0.01821333294113477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,8,128,1,float16,float16,0,0.017621333400408428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,4,128,1,fp8,fp8,0,0.01786133274435997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,40,8,128,1,fp8,fp8,0,0.017887999614079792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,1,128,1,float16,float16,0,0.4750880002975464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,1,128,1,float16,fp8,0,0.47496533393859863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,2,128,1,float16,float16,0,0.574282685915629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,1,128,1,fp8,fp8,0,0.4710986614227295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,2,128,1,float16,fp8,0,0.4790346622467041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,2,128,1,fp8,fp8,0,0.47626666227976483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,4,128,1,float16,float16,0,0.4862186511357625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,4,128,1,float16,fp8,0,0.4856746594111125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,4,128,1,fp8,fp8,0,0.4869653383890788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,8,128,1,float16,float16,0,0.5018826723098755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,8,128,1,float16,fp8,0,0.6381119887034098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,40,8,128,1,fp8,fp8,0,0.5022186835606893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,1,128,1,float16,float16,0,0.24048000574111938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,40,128,1,float16,float16,0,0.3302560051282247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,1,128,1,float16,fp8,0,0.24035199483235678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,40,128,1,float16,fp8,0,0.31255465745925903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,40,128,1,fp8,fp8,0,0.3086613416671753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,1,128,1,fp8,fp8,0,0.24236800273259482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,2,128,1,float16,float16,0,0.2837493419647217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,2,128,1,float16,fp8,0,0.24290132522583008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,2,128,1,fp8,fp8,0,0.24463999271392822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,4,128,1,float16,float16,0,0.24862933158874512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,4,128,1,fp8,fp8,0,0.2497333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,4,128,1,float16,fp8,0,0.24859732389450073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,8,128,1,float16,float16,0,0.2551413377126058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,8,128,1,float16,fp8,0,0.25381867090861004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,40,8,128,1,fp8,fp8,0,0.28878400723139447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,40,128,1,float16,float16,0,0.17427200078964233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,1,128,1,float16,float16,0,0.12866666913032532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,1,128,1,float16,fp8,0,0.12889066338539124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,2,128,1,float16,float16,0,0.1296266714731852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,40,128,1,float16,fp8,0,0.1649386684099833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,40,128,1,fp8,fp8,0,0.16338666280110678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,1,128,1,fp8,fp8,0,0.13152000308036804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,2,128,1,float16,fp8,0,0.14266133308410645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,2,128,1,fp8,fp8,0,0.1346879998842875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,4,128,1,float16,float16,0,0.1332480013370514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,4,128,1,float16,fp8,0,0.13265599807103476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,4,128,1,fp8,fp8,0,0.13487466176350912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,40,128,1,float16,float16,0,0.13387200236320496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,8,128,1,float16,fp8,0,0.13523733615875244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,40,128,1,float16,fp8,0,0.09181867043177287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,8,128,1,fp8,fp8,0,0.137855996688207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,40,8,128,1,float16,float16,0,0.13613333304723105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,1,128,1,float16,fp8,0,0.07188799977302551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,1,128,1,float16,float16,0,0.0719893326361974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,40,128,1,fp8,fp8,0,0.09300266702969869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,2,128,1,float16,fp8,0,0.07257066667079926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,1,128,1,fp8,fp8,0,0.07646400233109792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,2,128,1,float16,float16,0,0.07257066667079926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,2,128,1,fp8,fp8,0,0.07518933216730754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,4,128,1,float16,float16,0,0.07346133391062419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,4,128,1,float16,fp8,0,0.07338133454322815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,4,128,1,fp8,fp8,0,0.07706666489442189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,8,128,1,float16,float16,0,0.07611200213432312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,8,128,1,float16,fp8,0,0.0758133331934611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,40,128,1,float16,float16,0,0.05019199848175049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,1,128,1,float16,float16,0,0.04083200047413508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,40,8,128,1,fp8,fp8,0,0.08646399776140849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,1,128,1,float16,fp8,0,0.0412266676624616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,40,128,1,fp8,fp8,0,0.05187733471393585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,1,128,1,fp8,fp8,0,0.0421973317861557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,40,128,1,float16,fp8,0,0.04789866507053375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,2,128,1,float16,float16,0,0.041637333730856575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,2,128,1,float16,fp8,0,0.04132800052563349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,2,128,1,fp8,fp8,0,0.04247466723124186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,4,128,1,float16,float16,0,0.04205866654713949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,4,128,1,float16,fp8,0,0.042223999897638954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,4,128,1,fp8,fp8,0,0.044682666659355164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,8,128,1,float16,float16,0,0.04276266694068909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,8,128,1,fp8,fp8,0,0.04663999875386556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,40,8,128,1,float16,fp8,0,0.04320533573627472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,40,128,1,float16,float16,0,0.030181333422660828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,40,128,1,fp8,fp8,0,0.03331733246644338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,1,128,1,float16,float16,0,0.028912000358104706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,40,128,1,float16,fp8,0,0.029696000119050343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,2,128,1,float16,float16,0,0.028746667007605236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,1,128,1,float16,fp8,0,0.02887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,1,128,1,fp8,fp8,0,0.029765332738558452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,2,128,1,fp8,fp8,0,0.030085332691669464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,2,128,1,float16,fp8,0,0.02906133234500885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,4,128,1,float16,float16,0,0.02899733434120814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,4,128,1,float16,fp8,0,0.029103999336560566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,4,128,1,fp8,fp8,0,0.030479999879995983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,8,128,1,float16,float16,0,0.029253333806991577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,40,128,1,float16,float16,0,0.022917332748572033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,8,128,1,fp8,fp8,0,0.032442666590213776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,40,128,1,fp8,fp8,0,0.024890666206677754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,40,128,1,float16,fp8,0,0.02294933299223582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,40,8,128,1,float16,fp8,0,0.029525332152843475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,1,128,1,float16,float16,0,0.022495999932289124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,1,128,1,float16,fp8,0,0.022346665461858112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,1,128,1,fp8,fp8,0,0.023445333043734234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,2,128,1,float16,float16,0,0.021984001000722248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,8,128,1,float16,float16,0,0.02271466702222824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,4,128,1,fp8,fp8,0,0.023946667710940044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,2,128,1,float16,fp8,0,0.022687998910744984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,4,128,1,float16,fp8,0,0.022677332162857056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,8,128,1,float16,fp8,0,0.023584000766277313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,2,128,1,fp8,fp8,0,0.02372266600529353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,4,128,1,float16,float16,0,0.022522665560245514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,40,8,128,1,fp8,fp8,0,0.02422933280467987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,40,128,1,float16,float16,0,0.016837333639462788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,40,128,1,float16,fp8,0,0.01722666621208191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,40,128,1,fp8,fp8,0,0.018506667266289394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,1,128,1,float16,fp8,0,0.0164533331990242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,1,128,1,float16,float16,0,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,1,128,1,fp8,fp8,0,0.0174346665541331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,2,128,1,float16,fp8,0,0.01659199967980385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,2,128,1,fp8,fp8,0,0.017407999684413273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,4,128,1,float16,float16,0,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,2,128,1,float16,float16,0,0.0161013330022494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,4,128,1,fp8,fp8,0,0.018021332720915478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,4,128,1,float16,fp8,0,0.016778666526079178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,8,128,1,float16,float16,0,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,40,128,1,float16,float16,0,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,8,128,1,fp8,fp8,0,0.018005333840847015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,40,8,128,1,float16,fp8,0,0.01651200031240781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,40,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,1,128,1,float16,float16,0,0.015770666301250458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,2,128,1,float16,float16,0,0.015615999698638916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,1,128,1,float16,fp8,0,0.01613333324591319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,1,128,1,fp8,fp8,0,0.016762666404247284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,4,128,1,float16,float16,0,0.015557333827018738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,2,128,1,fp8,fp8,0,0.0170666662355264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,4,128,1,float16,fp8,0,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,40,128,1,fp8,fp8,0,0.01704000060757001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,2,128,1,float16,fp8,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,4,128,1,fp8,fp8,0,0.016842667013406754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,8,128,1,fp8,fp8,0,0.01684800038735072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,8,128,1,float16,float16,0,0.015594666202863058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,40,8,128,1,float16,fp8,0,0.016255999604860943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,1,128,1,float16,fp8,0,0.32235199213027954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,1,128,1,float16,float16,0,0.34956268469492596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,2,128,1,float16,float16,0,0.3261760075887044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,1,128,1,fp8,fp8,0,0.3768266836802165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,2,128,1,float16,fp8,0,0.32556267579396564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,2,128,1,fp8,fp8,0,0.3790293137232463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,4,128,1,float16,fp8,0,0.33238399028778076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,4,128,1,float16,float16,0,0.33316800991694134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,4,128,1,fp8,fp8,0,0.3845866521199544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,8,128,1,float16,fp8,0,0.3400213321050008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,8,128,1,float16,float16,0,0.39468268553415936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,40,128,1,float16,float16,0,0.20888533194859824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,40,128,1,float16,fp8,0,0.2010879913965861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,40,8,128,1,fp8,fp8,0,0.38649598757425946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,40,128,1,fp8,fp8,0,0.22703999280929565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,1,128,1,float16,fp8,0,0.16892266273498535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,1,128,1,float16,float16,0,0.16902933518091837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,1,128,1,fp8,fp8,0,0.19826134045918783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,2,128,1,float16,fp8,0,0.1698240041732788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,2,128,1,float16,float16,0,0.18458133935928345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,2,128,1,fp8,fp8,0,0.20207999149958292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,4,128,1,float16,float16,0,0.17341333627700806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,4,128,1,fp8,fp8,0,0.20400534073511759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,4,128,1,float16,fp8,0,0.17359467347462973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,8,128,1,float16,fp8,0,0.17639466126759848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,8,128,1,float16,float16,0,0.17735999822616577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,40,8,128,1,fp8,fp8,0,0.20287466049194336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,40,128,1,float16,float16,0,0.14364799857139587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,1,128,1,float16,float16,0,0.08957333366076152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,40,128,1,float16,fp8,0,0.10735467076301575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,1,128,1,float16,fp8,0,0.08958933750788371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,1,128,1,fp8,fp8,0,0.10889066259066264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,2,128,1,float16,float16,0,0.08975467085838318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,40,128,1,fp8,fp8,0,0.12215466300646464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,2,128,1,float16,fp8,0,0.09033067027727763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,4,128,1,float16,float16,0,0.09200533231099446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,2,128,1,fp8,fp8,0,0.1123306651910146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,4,128,1,float16,fp8,0,0.09256533781687419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,4,128,1,fp8,fp8,0,0.11152533690134685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,8,128,1,float16,float16,0,0.09595200419425964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,8,128,1,float16,fp8,0,0.09523199995358785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,40,128,1,float16,float16,0,0.05851200222969055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,40,128,1,float16,fp8,0,0.05653866628805796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,40,128,1,fp8,fp8,0,0.07849599917729695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,40,8,128,1,fp8,fp8,0,0.11307199796040852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,1,128,1,float16,float16,0,0.050016000866889954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,1,128,1,fp8,fp8,0,0.06030400097370148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,1,128,1,float16,fp8,0,0.05004266897837321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,2,128,1,float16,float16,0,0.05008533100287119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,2,128,1,float16,fp8,0,0.049813335140546165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,2,128,1,fp8,fp8,0,0.06038400034109751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,4,128,1,float16,float16,0,0.05116266508897146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,4,128,1,fp8,fp8,0,0.06225066880385081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,8,128,1,float16,float16,0,0.05193600058555603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,8,128,1,fp8,fp8,0,0.06369066735108693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,8,128,1,float16,fp8,0,0.05157333115736643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,40,4,128,1,float16,fp8,0,0.051141331593195595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,40,128,1,float16,float16,0,0.03254933406909307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,40,128,1,float16,fp8,0,0.033045334120591484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,1,128,1,float16,float16,0,0.03197333216667175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,1,128,1,float16,fp8,0,0.031658666829268135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,40,128,1,fp8,fp8,0,0.03978666663169861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,1,128,1,fp8,fp8,0,0.03639466563860575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,2,128,1,float16,float16,0,0.03151999910672506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,2,128,1,fp8,fp8,0,0.036576000352700554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,2,128,1,float16,fp8,0,0.031658666829268135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,4,128,1,float16,float16,0,0.03175999969244003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,4,128,1,float16,fp8,0,0.032042667269706726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,4,128,1,fp8,fp8,0,0.03746666759252548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,8,128,1,float16,fp8,0,0.03263466556866964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,40,128,1,float16,float16,0,0.02362666775782903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,8,128,1,fp8,fp8,0,0.03879466652870178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,40,128,1,fp8,fp8,0,0.029290666182835896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,40,8,128,1,float16,float16,0,0.03230933348337809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,1,128,1,float16,fp8,0,0.02380266785621643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,40,128,1,float16,fp8,0,0.023925334215164185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,1,128,1,float16,float16,0,0.023552000522613525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,2,128,1,float16,float16,0,0.02330133318901062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,1,128,1,fp8,fp8,0,0.026842666169007618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,2,128,1,fp8,fp8,0,0.027082666754722595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,2,128,1,float16,fp8,0,0.02369066576162974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,4,128,1,float16,float16,0,0.023610666394233704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,4,128,1,float16,fp8,0,0.023989332218964893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,8,128,1,float16,fp8,0,0.023919999599456787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,8,128,1,float16,float16,0,0.023578666150569916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,40,128,1,float16,float16,0,0.01893866683046023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,8,128,1,fp8,fp8,0,0.028229333460330963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,40,128,1,float16,fp8,0,0.019071999937295914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,40,128,1,fp8,fp8,0,0.021877333521842957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,40,4,128,1,fp8,fp8,0,0.027130665878454845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,1,128,1,float16,float16,0,0.018330667167901993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,1,128,1,float16,fp8,0,0.018426666657129925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,2,128,1,float16,float16,0,0.018320000420014065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,1,128,1,fp8,fp8,0,0.021130666136741638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,2,128,1,float16,fp8,0,0.018245333184798557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,2,128,1,fp8,fp8,0,0.021333334346612293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,4,128,1,float16,float16,0,0.01865600049495697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,4,128,1,fp8,fp8,0,0.021802666286627453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,8,128,1,float16,float16,0,0.01874133323629697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,8,128,1,float16,fp8,0,0.019189332922299702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,4,128,1,float16,fp8,0,0.018751999984184902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,40,8,128,1,fp8,fp8,0,0.021520001192887623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,40,128,1,float16,float16,0,0.015226667126019796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,40,128,1,fp8,fp8,0,0.018005333840847015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,40,128,1,float16,fp8,0,0.01578666642308235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,1,128,1,float16,fp8,0,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,1,128,1,float16,float16,0,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,1,128,1,fp8,fp8,0,0.01709866647919019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,2,128,1,float16,fp8,0,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,2,128,1,fp8,fp8,0,0.01728533332546552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,4,128,1,float16,float16,0,0.015072000523408255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,4,128,1,float16,fp8,0,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,2,128,1,float16,float16,0,0.014842666685581207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,4,128,1,fp8,fp8,0,0.017322666943073273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,8,128,1,float16,float16,0,0.01571200042963028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,8,128,1,float16,fp8,0,0.015743999431530636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,40,8,128,1,fp8,fp8,0,0.017077332983414333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,40,128,1,float16,float16,0,0.014576000471909841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,40,128,1,fp8,fp8,0,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,40,128,1,float16,fp8,0,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,1,128,1,float16,fp8,0,0.014741333822409311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,2,128,1,float16,float16,0,0.01462399959564209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,1,128,1,fp8,fp8,0,0.016496000190575916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,1,128,1,float16,float16,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,2,128,1,float16,fp8,0,0.015114666273196539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,2,128,1,fp8,fp8,0,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,4,128,1,fp8,fp8,0,0.01634666696190834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,4,128,1,float16,float16,0,0.014463999619086584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,4,128,1,float16,fp8,0,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,8,128,1,float16,float16,0,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,8,128,1,float16,fp8,0,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,40,8,128,1,fp8,fp8,0,0.01621333385507266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,1,128,1,float16,fp8,0,0.27376000086466473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,1,128,1,fp8,fp8,0,0.32625067234039307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,2,128,1,fp8,fp8,0,0.3290719985961914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,2,128,1,float16,fp8,0,0.28517866134643555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,1,128,1,float16,float16,0,0.27502934137980145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,4,128,1,float16,float16,0,0.27739200989405316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,2,128,1,float16,float16,0,0.2751946647961934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,4,128,1,fp8,fp8,0,0.3283733328183492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,4,128,1,float16,fp8,0,0.27664534250895184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,8,128,1,float16,float16,0,0.27903467416763306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,8,128,1,float16,fp8,0,0.27902400493621826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,40,8,128,1,fp8,fp8,0,0.34297064940134686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,40,128,1,float16,fp8,0,0.14850133657455444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,40,128,1,fp8,fp8,0,0.18676799535751343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,40,128,1,float16,float16,0,0.1520746648311615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,1,128,1,float16,float16,0,0.14358400305112204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,1,128,1,fp8,fp8,0,0.17190933227539062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,2,128,1,float16,fp8,0,0.14617066582043967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,1,128,1,float16,fp8,0,0.14341866970062256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,2,128,1,float16,float16,0,0.14339733123779297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,2,128,1,fp8,fp8,0,0.17217600345611572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,4,128,1,float16,fp8,0,0.1455573340257009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,4,128,1,float16,float16,0,0.14453333616256714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,4,128,1,fp8,fp8,0,0.17476266622543335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,8,128,1,float16,float16,0,0.14724266529083252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,8,128,1,float16,fp8,0,0.15370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,40,128,1,float16,float16,0,0.07963733375072479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,40,128,1,float16,fp8,0,0.07686399916807811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,1,128,1,float16,float16,0,0.07643199960390727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,40,8,128,1,fp8,fp8,0,0.17641599973042807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,40,128,1,fp8,fp8,0,0.10283733407656352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,1,128,1,float16,fp8,0,0.07708799839019775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,1,128,1,fp8,fp8,0,0.09342933694521587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,2,128,1,float16,float16,0,0.07740266621112823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,2,128,1,float16,fp8,0,0.0773173322280248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,2,128,1,fp8,fp8,0,0.09331732988357544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,4,128,1,fp8,fp8,0,0.09610133369763692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,8,128,1,float16,float16,0,0.07826133569081624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,8,128,1,float16,fp8,0,0.07827199995517731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,4,128,1,float16,float16,0,0.07759466767311096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,8,128,1,fp8,fp8,0,0.09995200236638387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,40,128,1,float16,float16,0,0.043696001172065735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,40,4,128,1,float16,fp8,0,0.07874666651089986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,40,128,1,float16,fp8,0,0.04344533383846283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,1,128,1,float16,float16,0,0.04557866851488749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,1,128,1,float16,fp8,0,0.045370668172836304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,2,128,1,float16,fp8,0,0.04582933088143667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,2,128,1,float16,float16,0,0.04562666515509287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,40,128,1,fp8,fp8,0,0.05618133147557577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,1,128,1,fp8,fp8,0,0.05444799860318502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,4,128,1,float16,float16,0,0.04607999821503957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,2,128,1,fp8,fp8,0,0.05415999889373779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,4,128,1,float16,fp8,0,0.0460746685663859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,4,128,1,fp8,fp8,0,0.05509866774082184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,8,128,1,float16,float16,0,0.04631466666857401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,40,128,1,float16,fp8,0,0.028570666909217834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,8,128,1,float16,fp8,0,0.046538665890693665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,1,128,1,float16,fp8,0,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,40,8,128,1,fp8,fp8,0,0.05618133147557577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,40,128,1,fp8,fp8,0,0.0360000009338061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,40,128,1,float16,float16,0,0.028773332635561626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,1,128,1,float16,float16,0,0.028650666276613872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,2,128,1,float16,float16,0,0.028736000259717304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,1,128,1,fp8,fp8,0,0.03422933320204417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,2,128,1,float16,fp8,0,0.02886933336655299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,2,128,1,fp8,fp8,0,0.03411199897527695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,4,128,1,float16,float16,0,0.0288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,4,128,1,float16,fp8,0,0.028832000990708668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,4,128,1,fp8,fp8,0,0.03469333300987879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,8,128,1,float16,fp8,0,0.029114666084448498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,8,128,1,float16,float16,0,0.028512001037597656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,40,8,128,1,fp8,fp8,0,0.035530666510264076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,40,128,1,float16,float16,0,0.02089066555102666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,40,128,1,float16,fp8,0,0.02125866711139679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,40,128,1,fp8,fp8,0,0.025589334468046825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,1,128,1,fp8,fp8,0,0.02499199906984965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,1,128,1,float16,fp8,0,0.020506666352351505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,2,128,1,float16,fp8,0,0.020970667401949566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,2,128,1,float16,float16,0,0.020303999384244282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,1,128,1,float16,float16,0,0.02075733368595441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,2,128,1,fp8,fp8,0,0.02497600018978119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,4,128,1,float16,float16,0,0.02067733307679494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,4,128,1,float16,fp8,0,0.020762667059898376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,4,128,1,fp8,fp8,0,0.025434667865435284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,8,128,1,float16,float16,0,0.020746666938066483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,8,128,1,float16,fp8,0,0.021061333517233532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,40,8,128,1,fp8,fp8,0,0.025514667232831318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,40,128,1,fp8,fp8,0,0.021333334346612293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,40,128,1,float16,fp8,0,0.01793066660563151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,1,128,1,float16,fp8,0,0.017866666118303936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,1,128,1,float16,float16,0,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,40,128,1,float16,float16,0,0.01730666682124138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,2,128,1,float16,float16,0,0.01759999990463257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,1,128,1,fp8,fp8,0,0.02073066681623459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,2,128,1,float16,fp8,0,0.017898666361967724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,2,128,1,fp8,fp8,0,0.02072000006834666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,4,128,1,float16,float16,0,0.017727999637524288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,4,128,1,float16,fp8,0,0.017973333597183228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,8,128,1,float16,fp8,0,0.018063999712467194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,4,128,1,fp8,fp8,0,0.02094399929046631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,8,128,1,fp8,fp8,0,0.02065066620707512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,40,128,1,float16,float16,0,0.0145066666106383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,40,128,1,float16,fp8,0,0.014746667196353277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,40,128,1,fp8,fp8,0,0.017029333859682083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,40,8,128,1,float16,float16,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,1,128,1,float16,fp8,0,0.01492799942692121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,2,128,1,fp8,fp8,0,0.016538667182127636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,1,128,1,fp8,fp8,0,0.016623999923467636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,2,128,1,float16,float16,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,1,128,1,float16,float16,0,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,2,128,1,float16,fp8,0,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,4,128,1,float16,float16,0,0.014458666245142618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,4,128,1,fp8,fp8,0,0.01716800034046173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,8,128,1,float16,float16,0,0.014389333625634512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,4,128,1,float16,fp8,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,8,128,1,fp8,fp8,0,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,40,8,128,1,float16,fp8,0,0.014970666418472925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,40,128,1,fp8,fp8,0,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,40,128,1,float16,float16,0,0.0144213338692983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,40,128,1,float16,fp8,0,0.014805333067973455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,1,128,1,float16,float16,0,0.014378666877746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,1,128,1,float16,fp8,0,0.01443733274936676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,2,128,1,float16,float16,0,0.014197333405415217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,2,128,1,float16,fp8,0,0.014874666929244995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,2,128,1,fp8,fp8,0,0.01655999943614006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,4,128,1,float16,float16,0,0.014394666999578476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,4,128,1,fp8,fp8,0,0.016688000410795212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,8,128,1,float16,float16,0,0.014080000420411428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,1,128,1,fp8,fp8,0,0.016143999993801117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,4,128,1,float16,fp8,0,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,8,128,1,fp8,fp8,0,0.01634666696190834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,40,8,128,1,float16,fp8,0,0.014912000546852747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,1,128,1,fp8,fp8,0,21.041034698486328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,1,128,1,float16,float16,0,32.896484375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,1,128,1,float16,fp8,0,32.549364725748696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,2,128,1,fp8,fp8,0,20.98864491780599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,2,128,1,float16,float16,0,33.48984018961588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,2,128,1,float16,fp8,0,32.503552754720054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,4,128,1,float16,float16,0,32.964281717936196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,4,128,1,float16,fp8,0,32.43814341227213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,1,128,1,float16,float16,0,16.224661509195965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,4,128,1,fp8,fp8,0,21.305552164713543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,8,128,1,fp8,fp8,0,21.24720001220703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,32,128,1,float16,float16,0,16.517274220784504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,8,128,1,float16,fp8,0,32.80304972330729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,32,8,128,1,float16,float16,0,33.356213887532554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,32,128,1,fp8,fp8,0,10.764811197916666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,32,128,1,float16,fp8,0,16.134928385416668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,1,128,1,float16,fp8,0,16.201615651448567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,1,128,1,fp8,fp8,0,10.803717295328775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,2,128,1,float16,float16,0,16.53280512491862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,2,128,1,float16,fp8,0,16.713947296142578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,2,128,1,fp8,fp8,0,10.675626118977865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,4,128,1,float16,float16,0,16.666576385498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,4,128,1,float16,fp8,0,16.42133331298828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,4,128,1,fp8,fp8,0,10.628885269165039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,8,128,1,float16,float16,0,16.25898615519206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,8,128,1,float16,fp8,0,16.58746083577474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,1,128,1,float16,float16,0,8.354661305745443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,1,128,1,float16,fp8,0,8.240837097167969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,1,128,1,fp8,fp8,0,5.231509208679199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,2,128,1,float16,float16,0,8.177290598551432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,32,8,128,1,fp8,fp8,0,10.82577641805013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,32,128,1,float16,fp8,0,8.144863764444986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,32,128,1,float16,float16,0,8.28167978922526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,2,128,1,float16,fp8,0,8.270224253336588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,32,128,1,fp8,fp8,0,5.351978937784831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,2,128,1,fp8,fp8,0,5.3593705495198565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,4,128,1,float16,fp8,0,8.363887786865234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,4,128,1,fp8,fp8,0,5.352336247762044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,4,128,1,float16,float16,0,8.21999486287435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,8,128,1,float16,float16,0,8.34610684712728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,8,128,1,fp8,fp8,0,5.42359987894694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,32,8,128,1,float16,fp8,0,8.444016138712565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,32,128,1,float16,float16,0,4.00765864054362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,1,128,1,float16,float16,0,4.107989311218262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,32,128,1,float16,fp8,0,4.049594561258952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,1,128,1,fp8,fp8,0,2.7444852193196616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,1,128,1,float16,fp8,0,3.990074793497721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,2,128,1,float16,float16,0,4.054330507914226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,2,128,1,fp8,fp8,0,2.7611306508382163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,2,128,1,float16,fp8,0,4.182469367980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,32,128,1,fp8,fp8,0,2.703551928202311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,4,128,1,float16,float16,0,4.211381276448567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,4,128,1,float16,fp8,0,4.199472109476726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,4,128,1,fp8,fp8,0,2.7542292277018228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,8,128,1,float16,fp8,0,4.1083574295043945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,8,128,1,fp8,fp8,0,2.78002134958903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,32,8,128,1,float16,float16,0,4.034063975016276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,1,128,1,fp8,fp8,0,12.427008310953775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,1,128,1,float16,float16,0,19.130308787027996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,1,128,1,float16,fp8,0,19.10413869222005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,2,128,1,float16,float16,0,18.838308970133465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,2,128,1,fp8,fp8,0,12.393866221110025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,2,128,1,float16,fp8,0,18.78670374552409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,4,128,1,float16,float16,0,19.45580800374349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,4,128,1,float16,fp8,0,18.749088287353516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,4,128,1,fp8,fp8,0,12.464827219645182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,1,128,1,float16,float16,0,9.583269119262695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,8,128,1,float16,float16,0,19.416980743408203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,8,128,1,float16,fp8,0,19.159435272216797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,32,8,128,1,fp8,fp8,0,12.385088602701822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,32,128,1,float16,float16,0,9.429962793986002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,1,128,1,float16,fp8,0,9.538666407267252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,32,128,1,float16,fp8,0,9.410256067911783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,32,128,1,fp8,fp8,0,6.392346700032552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,1,128,1,fp8,fp8,0,6.120421091715495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,2,128,1,float16,float16,0,9.569877624511719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,2,128,1,fp8,fp8,0,6.163605372111003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,4,128,1,float16,float16,0,9.544746398925781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,2,128,1,float16,fp8,0,9.485183715820312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,4,128,1,float16,fp8,0,9.487712224324545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,4,128,1,fp8,fp8,0,6.224112192789714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,8,128,1,float16,float16,0,9.74459711710612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,8,128,1,float16,fp8,0,9.449077606201172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,1,128,1,float16,float16,0,4.581274668375651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,1,128,1,float16,fp8,0,4.597637176513672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,32,8,128,1,fp8,fp8,0,6.309637069702148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,1,128,1,fp8,fp8,0,3.1165812810262046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,32,128,1,float16,float16,0,4.486639976501465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,32,128,1,float16,fp8,0,4.708837191263835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,2,128,1,float16,fp8,0,4.800271987915039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,32,128,1,fp8,fp8,0,3.1417385737101235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,2,128,1,float16,float16,0,4.601978619893392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,2,128,1,fp8,fp8,0,3.0917332967122397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,4,128,1,float16,float16,0,4.777903874715169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,4,128,1,float16,fp8,0,4.797562599182129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,4,128,1,fp8,fp8,0,3.1001386642456055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,8,128,1,float16,float16,0,4.773738543192546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,8,128,1,float16,fp8,0,4.6722720464070635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,32,8,128,1,fp8,fp8,0,3.1142346064249673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,32,128,1,float16,float16,0,2.3499306042989097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,32,128,1,float16,fp8,0,2.3477226893107095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,1,128,1,float16,float16,0,2.44539737701416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,32,128,1,fp8,fp8,0,1.6758027076721191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,1,128,1,fp8,fp8,0,1.6675732930501301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,1,128,1,float16,fp8,0,2.3743626276652017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,2,128,1,float16,float16,0,2.410378615061442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,2,128,1,float16,fp8,0,2.3780320485432944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,4,128,1,float16,float16,0,2.4083840052286782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,4,128,1,float16,fp8,0,2.391712029774984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,2,128,1,fp8,fp8,0,1.668933391571045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,4,128,1,fp8,fp8,0,1.6795412699381511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,8,128,1,float16,float16,0,2.4216906229654946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,8,128,1,fp8,fp8,0,1.6840373675028484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,32,8,128,1,float16,fp8,0,2.385109265645345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,1,128,1,fp8,fp8,0,8.938101450602213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,1,128,1,float16,float16,0,13.5525754292806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,1,128,1,float16,fp8,0,13.304890950520834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,2,128,1,float16,fp8,0,13.595509847005209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,2,128,1,fp8,fp8,0,8.842725118001303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,2,128,1,float16,float16,0,13.274575551350912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,4,128,1,float16,float16,0,13.472991943359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,4,128,1,float16,fp8,0,13.390975952148438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,1,128,1,float16,float16,0,6.596970876057942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,4,128,1,fp8,fp8,0,9.132554372151693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,8,128,1,float16,float16,0,13.61892318725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,8,128,1,float16,fp8,0,13.529652913411459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,32,8,128,1,fp8,fp8,0,9.074949264526367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,32,128,1,float16,float16,0,6.684965133666992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,32,128,1,fp8,fp8,0,4.395615895589192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,32,128,1,float16,fp8,0,6.502421061197917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,1,128,1,float16,fp8,0,6.574480056762695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,1,128,1,fp8,fp8,0,4.41374397277832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,2,128,1,float16,float16,0,6.641797383626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,2,128,1,fp8,fp8,0,4.393701235453288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,2,128,1,float16,fp8,0,6.660645167032878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,4,128,1,float16,float16,0,6.793525060017903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,4,128,1,float16,fp8,0,6.829333623250325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,4,128,1,fp8,fp8,0,4.352725346883138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,8,128,1,float16,float16,0,6.712080001831055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,8,128,1,float16,fp8,0,6.864704132080078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,1,128,1,float16,float16,0,3.2651198705037436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,1,128,1,float16,fp8,0,3.282405217488607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,1,128,1,fp8,fp8,0,2.242506663004557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,2,128,1,float16,float16,0,3.291920026143392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,32,8,128,1,fp8,fp8,0,4.410218556722005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,32,128,1,float16,float16,0,3.336517333984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,32,128,1,float16,fp8,0,3.211423873901367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,32,128,1,fp8,fp8,0,2.2684426307678223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,2,128,1,float16,fp8,0,3.3684266408284507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,2,128,1,fp8,fp8,0,2.2463733355204263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,4,128,1,float16,float16,0,3.2610934575398765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,4,128,1,float16,fp8,0,3.2627731959025064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,4,128,1,fp8,fp8,0,2.2540480295817056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,8,128,1,float16,float16,0,3.3362719217936196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,8,128,1,float16,fp8,0,3.2444960276285806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,32,8,128,1,fp8,fp8,0,2.267120043436686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,32,128,1,float16,fp8,0,1.6958932876586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,32,128,1,float16,float16,0,1.7233813603719075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,1,128,1,float16,float16,0,1.7524159749348958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,1,128,1,float16,fp8,0,1.7203893661499023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,1,128,1,fp8,fp8,0,1.2296586831410725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,2,128,1,float16,float16,0,1.7453120549519856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,32,128,1,fp8,fp8,0,1.2256639798482258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,2,128,1,float16,fp8,0,1.7290186882019043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,4,128,1,float16,float16,0,1.7507519721984863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,4,128,1,float16,fp8,0,1.7351519266764324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,2,128,1,fp8,fp8,0,1.2292053699493408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,8,128,1,float16,float16,0,1.7749546368916829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,4,128,1,fp8,fp8,0,1.232805331548055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,8,128,1,float16,fp8,0,1.73199462890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,32,8,128,1,fp8,fp8,0,1.2374826272328694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,1,128,1,float16,float16,0,17.824469248453777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,1,128,1,fp8,fp8,0,11.983445485432943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,1,128,1,float16,fp8,0,17.632181803385418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,2,128,1,float16,fp8,0,17.905466715494793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,2,128,1,float16,float16,0,17.5416742960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,2,128,1,fp8,fp8,0,11.838768005371094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,4,128,1,float16,float16,0,17.77035140991211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,4,128,1,float16,fp8,0,17.468922932942707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,1,128,1,float16,float16,0,8.810288111368815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,4,128,1,fp8,fp8,0,11.866474151611328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,8,128,1,float16,fp8,0,17.731104532877605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,8,128,1,float16,float16,0,18.224506378173828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,1,128,1,float16,fp8,0,8.833242416381836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,32,8,128,1,fp8,fp8,0,12.200735727945963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,32,128,1,float16,float16,0,8.689898808797201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,32,128,1,float16,fp8,0,8.883557637532553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,1,128,1,fp8,fp8,0,5.811759948730469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,32,128,1,fp8,fp8,0,5.950143814086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,2,128,1,float16,fp8,0,8.986986796061197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,2,128,1,float16,float16,0,8.861194610595703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,2,128,1,fp8,fp8,0,5.7103627522786455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,4,128,1,float16,float16,0,8.949130376180014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,4,128,1,float16,fp8,0,8.761130650838217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,4,128,1,fp8,fp8,0,5.815776189168294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,8,128,1,float16,float16,0,8.990687688191732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,8,128,1,float16,fp8,0,8.872400283813477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,32,8,128,1,fp8,fp8,0,6.02130126953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,32,128,1,float16,float16,0,4.309770584106445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,1,128,1,float16,fp8,0,4.154538790384929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,1,128,1,fp8,fp8,0,2.9072373708089194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,1,128,1,float16,float16,0,4.191983858744304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,2,128,1,float16,float16,0,4.209520022074382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,32,128,1,float16,fp8,0,4.3002878824869795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,32,128,1,fp8,fp8,0,3.0457919438680015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,2,128,1,float16,fp8,0,4.32256539662679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,2,128,1,fp8,fp8,0,2.9178988138834634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,4,128,1,float16,fp8,0,4.314618746439616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,4,128,1,float16,float16,0,4.359509468078613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,4,128,1,fp8,fp8,0,2.92682679494222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,1,128,1,float16,float16,0,2.1858293215433755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,8,128,1,float16,float16,0,4.428869247436523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,8,128,1,fp8,fp8,0,2.978800137837728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,1,128,1,float16,fp8,0,2.1547306378682456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,32,8,128,1,float16,fp8,0,4.183882713317871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,32,128,1,float16,float16,0,2.2936906814575195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,32,128,1,float16,fp8,0,2.169637362162272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,1,128,1,fp8,fp8,0,1.5341280301411946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,32,128,1,fp8,fp8,0,1.5640427271525066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,2,128,1,float16,fp8,0,2.198901335398356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,2,128,1,fp8,fp8,0,1.5359892845153809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,2,128,1,float16,float16,0,2.1914240519205728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,4,128,1,float16,float16,0,2.192629337310791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,4,128,1,float16,fp8,0,2.193269411722819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,4,128,1,fp8,fp8,0,1.5438346862792969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,1,128,1,float16,float16,0,1.196837345759074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,8,128,1,float16,float16,0,2.201258659362793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,1,128,1,float16,fp8,0,1.172165314356486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,32,128,1,float16,float16,0,1.1704479853312175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,8,128,1,fp8,fp8,0,1.5582027435302734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,32,128,1,fp8,fp8,0,0.8600800037384033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,32,8,128,1,float16,fp8,0,2.1857120196024575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,1,128,1,fp8,fp8,0,0.8633386294047037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,2,128,1,float16,fp8,0,1.174181302388509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,2,128,1,float16,float16,0,1.1891466776529949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,2,128,1,fp8,fp8,0,0.85698135693868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,32,128,1,float16,fp8,0,1.1663466294606526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,4,128,1,float16,float16,0,1.205194632212321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,4,128,1,float16,fp8,0,1.1737386385599773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,4,128,1,fp8,fp8,0,0.8578933080037435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,8,128,1,float16,float16,0,1.2087360223134358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,8,128,1,float16,fp8,0,1.180399974187215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,32,8,128,1,fp8,fp8,0,0.8623200257619222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,1,128,1,fp8,fp8,0,7.043157577514648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,1,128,1,float16,fp8,0,10.31382942199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,2,128,1,float16,float16,0,10.580741246541342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,1,128,1,float16,float16,0,10.342970530192057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,2,128,1,fp8,fp8,0,7.296127955118815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,4,128,1,float16,float16,0,10.607653299967447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,4,128,1,float16,fp8,0,10.524645487467447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,2,128,1,float16,fp8,0,10.45733896891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,1,128,1,float16,float16,0,5.069519996643066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,4,128,1,fp8,fp8,0,7.196821212768555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,8,128,1,float16,fp8,0,10.486245473225912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,8,128,1,float16,float16,0,10.548282623291016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,32,128,1,float16,float16,0,5.454847971598308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,1,128,1,float16,fp8,0,5.137397448221843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,32,8,128,1,fp8,fp8,0,7.116133371988933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,32,128,1,float16,fp8,0,5.221856117248535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,1,128,1,fp8,fp8,0,3.491957346598307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,32,128,1,fp8,fp8,0,3.588149388631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,2,128,1,float16,float16,0,4.940650622049968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,2,128,1,fp8,fp8,0,3.6039787928263345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,2,128,1,float16,fp8,0,4.88536008199056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,4,128,1,float16,float16,0,4.98307736714681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,4,128,1,fp8,fp8,0,3.5064051946004233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,4,128,1,float16,fp8,0,4.98197873433431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,8,128,1,float16,float16,0,5.081391970316569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,1,128,1,float16,float16,0,2.5266613960266113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,1,128,1,float16,fp8,0,2.479402701059977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,1,128,1,fp8,fp8,0,1.7915573120117188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,8,128,1,float16,fp8,0,5.265679995218913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,32,128,1,float16,float16,0,2.7573814392089844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,32,8,128,1,fp8,fp8,0,3.534031867980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,32,128,1,float16,fp8,0,2.531930605570475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,2,128,1,float16,float16,0,2.5362399419148765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,2,128,1,float16,fp8,0,2.486570676167806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,32,128,1,fp8,fp8,0,1.8631572723388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,2,128,1,fp8,fp8,0,1.7969066301981609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,4,128,1,float16,float16,0,2.530597368876139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,4,128,1,float16,fp8,0,2.5404052734375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,4,128,1,fp8,fp8,0,1.7995786666870117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,8,128,1,float16,float16,0,2.5464960734049478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,8,128,1,float16,fp8,0,2.5198453267415366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,1,128,1,float16,float16,0,1.3463412920633953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,1,128,1,float16,fp8,0,1.3078239758809407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,32,8,128,1,fp8,fp8,0,1.8188586235046387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,1,128,1,fp8,fp8,0,0.9580480257670084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,32,128,1,float16,float16,0,1.3419946034749348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,32,128,1,float16,fp8,0,1.3451733589172363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,2,128,1,float16,float16,0,1.334426720937093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,2,128,1,float16,fp8,0,1.3286666870117188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,2,128,1,fp8,fp8,0,0.9589973290761312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,4,128,1,float16,float16,0,1.3338826497395833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,4,128,1,fp8,fp8,0,0.9641653696695963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,4,128,1,float16,fp8,0,1.3160160382588704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,32,128,1,fp8,fp8,0,0.9862986405690511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,8,128,1,float16,float16,0,1.358847935994466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,8,128,1,fp8,fp8,0,0.990768035252889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,32,8,128,1,float16,fp8,0,1.3253440062204997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,32,128,1,float16,float16,0,0.733903964360555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,1,128,1,float16,float16,0,0.7401813666025797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,32,128,1,float16,fp8,0,0.7237919966379801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,1,128,1,float16,fp8,0,0.7272213300069174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,1,128,1,fp8,fp8,0,0.5175466537475586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,32,128,1,fp8,fp8,0,0.5438826481501261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,2,128,1,float16,float16,0,0.7474186420440674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,2,128,1,float16,fp8,0,0.7292906443277994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,2,128,1,fp8,fp8,0,0.521232008934021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,4,128,1,float16,float16,0,0.7426933447519938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,4,128,1,float16,fp8,0,0.7292106946309408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,4,128,1,fp8,fp8,0,0.521888017654419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,8,128,1,float16,float16,0,0.7488906383514404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,8,128,1,fp8,fp8,0,0.5362826585769653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,32,8,128,1,float16,fp8,0,0.7330079873402914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,1,128,1,float16,float16,0,10.161200205485025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,1,128,1,fp8,fp8,0,7.050314585367839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,2,128,1,float16,fp8,0,10.031717300415039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,1,128,1,float16,fp8,0,10.08299191792806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,2,128,1,float16,float16,0,10.162053426106771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,2,128,1,fp8,fp8,0,7.135082880655925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,4,128,1,float16,float16,0,10.758319854736328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,4,128,1,float16,fp8,0,10.192991892496744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,1,128,1,float16,float16,0,5.045909245808919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,4,128,1,fp8,fp8,0,7.1215254465738935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,1,128,1,float16,fp8,0,4.838234583536784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,8,128,1,float16,float16,0,10.399434407552084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,8,128,1,float16,fp8,0,10.28599484761556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,32,8,128,1,fp8,fp8,0,7.162298838297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,32,128,1,fp8,fp8,0,4.087295850118001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,32,128,1,float16,fp8,0,5.185301462809245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,1,128,1,fp8,fp8,0,3.5112692515055337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,32,128,1,float16,float16,0,5.0711361567179365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,2,128,1,float16,float16,0,4.989813486735026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,2,128,1,fp8,fp8,0,3.5220959981282554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,2,128,1,float16,fp8,0,4.855104128519694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,4,128,1,float16,float16,0,5.077199935913086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,4,128,1,float16,fp8,0,5.175354639689128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,4,128,1,fp8,fp8,0,3.612325350443522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,8,128,1,float16,float16,0,5.107237180074056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,8,128,1,float16,fp8,0,4.981888135274251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,1,128,1,float16,float16,0,2.496613343556722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,1,128,1,float16,fp8,0,2.487834612528483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,32,8,128,1,fp8,fp8,0,3.5922559102376304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,1,128,1,fp8,fp8,0,1.7832959493001301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,32,128,1,float16,fp8,0,2.5045973459879556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,32,128,1,float16,float16,0,2.540170669555664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,2,128,1,float16,float16,0,2.438255945841471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,32,128,1,fp8,fp8,0,1.8954720497131348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,2,128,1,float16,fp8,0,2.454319953918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,2,128,1,fp8,fp8,0,1.7882134119669597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,4,128,1,float16,fp8,0,2.4132374127705893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,4,128,1,float16,float16,0,2.4610506693522134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,4,128,1,fp8,fp8,0,1.79857603708903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,8,128,1,float16,fp8,0,2.423466682434082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,8,128,1,float16,float16,0,2.4694347381591797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,32,128,1,float16,float16,0,1.4392266273498535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,32,8,128,1,fp8,fp8,0,1.8244214057922363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,1,128,1,float16,fp8,0,1.2425333658854167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,32,128,1,float16,fp8,0,1.3039466540018718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,1,128,1,float16,float16,0,1.2678453127543132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,1,128,1,fp8,fp8,0,0.9330026308695475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,2,128,1,float16,float16,0,1.2672800223032634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,32,128,1,fp8,fp8,0,0.9970080057779948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,2,128,1,float16,fp8,0,1.2712960243225098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,2,128,1,fp8,fp8,0,0.936730702718099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,4,128,1,float16,fp8,0,1.2661759853363037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,4,128,1,float16,float16,0,1.2738773028055828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,4,128,1,fp8,fp8,0,0.944917360941569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,8,128,1,float16,float16,0,1.2798933188120525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,8,128,1,float16,fp8,0,1.2591253121693928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,1,128,1,float16,float16,0,0.6833279927571615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,1,128,1,float16,fp8,0,0.6706346670786539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,32,8,128,1,fp8,fp8,0,0.9592639605204264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,32,128,1,float16,float16,0,0.7491679986317953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,1,128,1,fp8,fp8,0,0.5135413408279419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,32,128,1,float16,fp8,0,0.6833439668019613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,2,128,1,float16,float16,0,0.6853333314259847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,32,128,1,fp8,fp8,0,0.5421013434727987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,2,128,1,float16,fp8,0,0.6716053485870361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,2,128,1,fp8,fp8,0,0.5152693192164103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,4,128,1,float16,float16,0,0.6896906693776449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,4,128,1,fp8,fp8,0,0.5171626806259155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,4,128,1,float16,fp8,0,0.6878720124562582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,8,128,1,float16,float16,0,0.6903253396352133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,8,128,1,fp8,fp8,0,0.5237546761830648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,32,8,128,1,float16,fp8,0,0.6780213514963785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,32,128,1,float16,float16,0,0.3973333438237508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,1,128,1,float16,float16,0,0.3945759932200114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,1,128,1,float16,fp8,0,0.39006932576497394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,32,128,1,float16,fp8,0,0.3908853530883789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,32,128,1,fp8,fp8,0,0.30398933092753094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,1,128,1,fp8,fp8,0,0.28536534309387207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,2,128,1,float16,float16,0,0.3972959915796916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,2,128,1,float16,fp8,0,0.38736534118652344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,2,128,1,fp8,fp8,0,0.28464533885320026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,4,128,1,float16,float16,0,0.39769065380096436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,4,128,1,float16,fp8,0,0.3943306605021159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,4,128,1,fp8,fp8,0,0.2871840000152588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,8,128,1,float16,float16,0,0.40066667397816974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,8,128,1,float16,fp8,0,0.39096001784006756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,32,8,128,1,fp8,fp8,0,0.290175994237264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,1,128,1,fp8,fp8,0,4.425840059916179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,1,128,1,float16,float16,0,6.0737654368082685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,1,128,1,float16,fp8,0,5.957621256510417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,2,128,1,fp8,fp8,0,4.450826644897461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,2,128,1,float16,float16,0,6.519301096598308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,2,128,1,float16,fp8,0,6.111930847167969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,4,128,1,float16,float16,0,6.161354700724284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,4,128,1,float16,fp8,0,6.170277277628581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,1,128,1,float16,float16,0,3.0398454666137695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,4,128,1,fp8,fp8,0,4.478634516398112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,1,128,1,float16,fp8,0,2.985018730163574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,8,128,1,fp8,fp8,0,4.57151476542155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,8,128,1,float16,float16,0,6.257551829020183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,32,128,1,float16,float16,0,3.5923999150594077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,32,8,128,1,float16,fp8,0,6.085701624552409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,32,128,1,float16,fp8,0,3.1231199900309243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,32,128,1,fp8,fp8,0,2.4110347429911294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,1,128,1,fp8,fp8,0,2.2349440256754556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,2,128,1,float16,fp8,0,2.9266185760498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,2,128,1,float16,float16,0,3.031418800354004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,2,128,1,fp8,fp8,0,2.244485378265381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,4,128,1,float16,float16,0,3.193135897318522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,4,128,1,fp8,fp8,0,2.2580746014912925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,4,128,1,float16,fp8,0,2.9654668172200522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,8,128,1,float16,float16,0,3.0350828170776367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,1,128,1,float16,float16,0,1.5182933807373047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,1,128,1,float16,fp8,0,1.4834133783976238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,8,128,1,float16,fp8,0,2.9606345494588218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,1,128,1,fp8,fp8,0,1.1398399670918782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,32,8,128,1,fp8,fp8,0,2.2958399454752603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,32,128,1,float16,float16,0,1.6272053718566895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,2,128,1,float16,float16,0,1.5230080286661785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,2,128,1,float16,fp8,0,1.4874986012776692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,32,128,1,fp8,fp8,0,1.3696533838907878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,32,128,1,float16,fp8,0,1.575424035390218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,2,128,1,fp8,fp8,0,1.1452159881591797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,4,128,1,float16,float16,0,1.5261653264363606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,4,128,1,float16,fp8,0,1.4982080459594727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,4,128,1,fp8,fp8,0,1.1526933511098225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,8,128,1,float16,float16,0,1.543392022450765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,8,128,1,fp8,fp8,0,1.2142773469289143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,32,8,128,1,float16,fp8,0,1.5104427337646484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,1,128,1,float16,float16,0,0.8005812962849935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,1,128,1,float16,fp8,0,0.7845813433329264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,32,128,1,float16,float16,0,0.829045295715332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,32,128,1,float16,fp8,0,0.8120266596476237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,32,128,1,fp8,fp8,0,0.6605333487192789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,2,128,1,float16,float16,0,0.8233280181884766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,2,128,1,float16,fp8,0,0.7824959754943848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,1,128,1,fp8,fp8,0,0.6063253482182821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,2,128,1,fp8,fp8,0,0.610421339670817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,4,128,1,float16,float16,0,0.8082666397094727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,4,128,1,fp8,fp8,0,0.6144746541976929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,4,128,1,float16,fp8,0,0.7860906918843588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,8,128,1,float16,float16,0,0.8082453409830729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,1,128,1,float16,float16,0,0.43873600165049237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,8,128,1,float16,fp8,0,0.8236160278320312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,1,128,1,float16,fp8,0,0.4298880100250244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,32,8,128,1,fp8,fp8,0,0.6239466667175293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,32,128,1,float16,float16,0,0.4497493505477905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,32,128,1,float16,fp8,0,0.44355201721191406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,32,128,1,fp8,fp8,0,0.3664799928665161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,1,128,1,fp8,fp8,0,0.32629332939783734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,2,128,1,float16,float16,0,0.44073065121968585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,2,128,1,float16,fp8,0,0.43993067741394043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,2,128,1,fp8,fp8,0,0.32842133442560834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,4,128,1,float16,float16,0,0.4426240126291911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,4,128,1,float16,fp8,0,0.43334933121999103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,4,128,1,fp8,fp8,0,0.3303413391113281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,8,128,1,float16,float16,0,0.446176012357076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,8,128,1,float16,fp8,0,0.43620800971984863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,1,128,1,float16,float16,0,0.23754133780797324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,32,8,128,1,fp8,fp8,0,0.336847980817159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,1,128,1,float16,fp8,0,0.23213332891464233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,32,128,1,float16,float16,0,0.26652799050013226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,1,128,1,fp8,fp8,0,0.1909653345743815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,32,128,1,fp8,fp8,0,0.2058080037434896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,2,128,1,float16,float16,0,0.23779199520746866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,32,128,1,float16,fp8,0,0.24146666129430136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,2,128,1,float16,fp8,0,0.23150400320688883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,2,128,1,fp8,fp8,0,0.19225066900253296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,4,128,1,float16,fp8,0,0.2351413369178772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,4,128,1,float16,float16,0,0.2389919956525167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,4,128,1,fp8,fp8,0,0.19347200791041055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,8,128,1,float16,float16,0,0.23980265855789185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,8,128,1,float16,fp8,0,0.23473066091537476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,32,8,128,1,fp8,fp8,0,0.19584532578786215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,1,128,1,fp8,fp8,0,5.3650557200113935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,1,128,1,float16,float16,0,6.397861480712891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,1,128,1,float16,fp8,0,6.127930959065755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,2,128,1,float16,fp8,0,6.360282897949219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,2,128,1,float16,float16,0,6.430869420369466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,2,128,1,fp8,fp8,0,4.915701230367024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,4,128,1,float16,float16,0,6.472906748453776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,4,128,1,float16,fp8,0,6.296784083048503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,1,128,1,float16,float16,0,3.1396961212158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,4,128,1,fp8,fp8,0,5.44436772664388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,8,128,1,float16,float16,0,6.564794540405273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,8,128,1,fp8,fp8,0,5.034874598185222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,1,128,1,float16,fp8,0,3.0244318644205728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,32,128,1,fp8,fp8,0,2.7084213892618814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,32,8,128,1,float16,fp8,0,6.232378641764323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,32,128,1,float16,float16,0,3.415856043497721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,32,128,1,float16,fp8,0,3.3035732905069985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,1,128,1,fp8,fp8,0,2.6411892573038735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,2,128,1,float16,float16,0,3.145440101623535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,2,128,1,float16,fp8,0,3.066842714945475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,2,128,1,fp8,fp8,0,2.4506400426228843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,4,128,1,fp8,fp8,0,2.472229321797689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,4,128,1,float16,float16,0,3.156815846761068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,4,128,1,float16,fp8,0,3.05133851369222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,8,128,1,float16,float16,0,3.1984532674153647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,1,128,1,float16,float16,0,1.5570933024088542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,8,128,1,float16,fp8,0,3.4186827341715493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,1,128,1,float16,fp8,0,1.5148213704427083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,1,128,1,fp8,fp8,0,1.2248000303904216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,32,8,128,1,fp8,fp8,0,2.5135520299275718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,2,128,1,float16,float16,0,1.5616587003072102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,32,128,1,float16,float16,0,1.7238720258076985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,2,128,1,float16,fp8,0,1.588362693786621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,32,128,1,fp8,fp8,0,1.3727787335713704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,2,128,1,fp8,fp8,0,1.2315200169881184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,32,128,1,float16,fp8,0,1.6628212928771973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,4,128,1,float16,float16,0,1.5750826199849446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,4,128,1,float16,fp8,0,1.531285285949707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,4,128,1,fp8,fp8,0,1.248095989227295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,8,128,1,float16,fp8,0,1.654527982076009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,8,128,1,float16,float16,0,1.6063040097554524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,32,128,1,float16,float16,0,0.8685812950134277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,32,8,128,1,fp8,fp8,0,1.2688799699147542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,1,128,1,float16,float16,0,0.8090879917144775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,1,128,1,float16,fp8,0,0.7864000002543131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,32,128,1,float16,fp8,0,0.8499093055725098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,32,128,1,fp8,fp8,0,0.7141919930775961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,2,128,1,float16,float16,0,0.8132747014363607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,2,128,1,float16,fp8,0,0.787775993347168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,2,128,1,fp8,fp8,0,0.6415359973907471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,1,128,1,fp8,fp8,0,0.661407987276713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,4,128,1,float16,float16,0,0.8158666292826334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,4,128,1,float16,fp8,0,0.7937226295471191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,8,128,1,float16,float16,0,0.8245866298675537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,4,128,1,fp8,fp8,0,0.6476426521937052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,8,128,1,float16,fp8,0,0.80731201171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,1,128,1,float16,float16,0,0.4336959918340047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,32,8,128,1,fp8,fp8,0,0.6576693455378214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,32,128,1,float16,float16,0,0.4591199954350789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,32,128,1,fp8,fp8,0,0.38603734970092773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,1,128,1,float16,fp8,0,0.4227466583251953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,32,128,1,float16,fp8,0,0.5069973468780518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,1,128,1,fp8,fp8,0,0.3472213347752889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,2,128,1,float16,float16,0,0.43507734934488934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,2,128,1,float16,fp8,0,0.4238186677296956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,4,128,1,float16,float16,0,0.437824010848999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,4,128,1,float16,fp8,0,0.4259359836578369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,4,128,1,fp8,fp8,0,0.35156798362731934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,8,128,1,float16,float16,0,0.46746667226155597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,8,128,1,float16,fp8,0,0.43162135283152264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,2,128,1,fp8,fp8,0,0.34858667850494385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,32,8,128,1,fp8,fp8,0,0.3569120168685913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,1,128,1,float16,float16,0,0.2454026738802592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,32,128,1,float16,float16,0,0.2564000089963277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,32,128,1,float16,fp8,0,0.25242666403452557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,32,128,1,fp8,fp8,0,0.21371734142303467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,1,128,1,float16,fp8,0,0.24296534061431885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,1,128,1,fp8,fp8,0,0.18943466742833456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,2,128,1,fp8,fp8,0,0.18966400623321533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,2,128,1,float16,float16,0,0.2462666630744934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,4,128,1,float16,float16,0,0.24797866741816202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,4,128,1,float16,fp8,0,0.24228266874949136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,4,128,1,fp8,fp8,0,0.19132800896962485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,2,128,1,float16,fp8,0,0.24050132433573404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,8,128,1,float16,float16,0,0.26156800985336304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,8,128,1,float16,fp8,0,0.24503467480341592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,32,8,128,1,fp8,fp8,0,0.1946400006612142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,1,128,1,float16,float16,0,0.13527466853459677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,32,128,1,float16,fp8,0,0.13945600390434265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,1,128,1,float16,fp8,0,0.13134933511416116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,32,128,1,fp8,fp8,0,0.12613333264986673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,1,128,1,fp8,fp8,0,0.11354133486747742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,32,128,1,float16,float16,0,0.14317333698272705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,2,128,1,float16,fp8,0,0.13199999928474426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,2,128,1,fp8,fp8,0,0.11415466666221619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,2,128,1,float16,float16,0,0.13523200154304504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,4,128,1,float16,float16,0,0.13621333241462708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,4,128,1,float16,fp8,0,0.13220266501108804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,4,128,1,fp8,fp8,0,0.11648000280062358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,8,128,1,float16,float16,0,0.13918399810791016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,8,128,1,fp8,fp8,0,0.11784000198046367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,32,8,128,1,float16,fp8,0,0.13366933663686117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,1,128,1,float16,float16,0,4.121157328287761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,1,128,1,fp8,fp8,0,3.3338187535603843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,1,128,1,float16,fp8,0,3.9293225606282554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,2,128,1,float16,float16,0,4.168479919433594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,2,128,1,fp8,fp8,0,3.6161066691080728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,2,128,1,float16,fp8,0,3.943023999532064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,4,128,1,float16,float16,0,4.172591845194499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,4,128,1,float16,fp8,0,3.96455987294515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,1,128,1,float16,float16,0,2.0279946327209473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,4,128,1,fp8,fp8,0,3.3538614908854165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,8,128,1,float16,float16,0,4.234352111816406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,8,128,1,fp8,fp8,0,3.3988958994547525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,32,128,1,float16,fp8,0,2.179317315419515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,32,8,128,1,float16,fp8,0,4.025322596232097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,1,128,1,float16,fp8,0,1.9431734085083008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,32,128,1,fp8,fp8,0,1.8625226020812988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,1,128,1,fp8,fp8,0,1.6436479886372883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,32,128,1,float16,float16,0,2.680053393046061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,2,128,1,float16,float16,0,2.0314292907714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,2,128,1,float16,fp8,0,1.9602452913920085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,2,128,1,fp8,fp8,0,1.654197374979655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,4,128,1,float16,float16,0,2.0534133911132812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,4,128,1,float16,fp8,0,1.9630187352498372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,4,128,1,fp8,fp8,0,1.6661972999572754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,8,128,1,float16,float16,0,2.0909172693888345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,1,128,1,float16,float16,0,1.0160693327585857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,8,128,1,float16,fp8,0,2.2317439715067544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,1,128,1,float16,fp8,0,0.985706647237142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,32,8,128,1,fp8,fp8,0,1.7098347345987956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,1,128,1,fp8,fp8,0,0.8321759700775146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,32,128,1,float16,float16,0,1.1420693397521973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,32,128,1,float16,fp8,0,1.1029333273569744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,2,128,1,float16,float16,0,1.020896037419637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,32,128,1,fp8,fp8,0,0.9507413705190023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,2,128,1,float16,fp8,0,1.0345813433329265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,2,128,1,fp8,fp8,0,0.837445338567098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,4,128,1,float16,float16,0,1.0255786577860515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,4,128,1,float16,fp8,0,0.9960959752400717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,4,128,1,fp8,fp8,0,0.8437973658243815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,8,128,1,float16,float16,0,1.0416213671366374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,1,128,1,float16,float16,0,0.5332320133845011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,1,128,1,float16,fp8,0,0.5163360039393107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,8,128,1,float16,fp8,0,1.0101226965586345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,32,8,128,1,fp8,fp8,0,0.862938642501831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,32,128,1,float16,float16,0,0.6953919728597006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,1,128,1,fp8,fp8,0,0.4399679899215698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,32,128,1,float16,fp8,0,0.568560004234314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,2,128,1,float16,float16,0,0.5350879828135172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,32,128,1,fp8,fp8,0,0.49914666016896564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,2,128,1,float16,fp8,0,0.5186826785405477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,2,128,1,fp8,fp8,0,0.440287987391154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,4,128,1,fp8,fp8,0,0.44572265942891437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,8,128,1,float16,float16,0,0.5453600088755289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,4,128,1,float16,fp8,0,0.5487519900004069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,4,128,1,float16,float16,0,0.5388799905776978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,8,128,1,float16,fp8,0,0.529914657274882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,32,8,128,1,fp8,fp8,0,0.45634134610493976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,32,128,1,float16,float16,0,0.31160000960032147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,1,128,1,float16,float16,0,0.2908693353335063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,1,128,1,float16,fp8,0,0.2911733388900757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,2,128,1,float16,float16,0,0.2918613354365031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,32,128,1,fp8,fp8,0,0.2839999993642171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,32,128,1,float16,fp8,0,0.30322132507960003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,1,128,1,fp8,fp8,0,0.23131734132766724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,2,128,1,float16,fp8,0,0.28334399064381915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,2,128,1,fp8,fp8,0,0.23290133476257324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,4,128,1,float16,float16,0,0.29366399844487506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,4,128,1,fp8,fp8,0,0.23483733336130777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,8,128,1,float16,float16,0,0.29709333181381226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,4,128,1,float16,fp8,0,0.29710400104522705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,8,128,1,float16,fp8,0,0.28807999690373737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,32,128,1,float16,float16,0,0.1774453322092692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,32,8,128,1,fp8,fp8,0,0.2405653397242228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,32,128,1,float16,fp8,0,0.17360534270604452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,1,128,1,float16,float16,0,0.1590559979279836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,32,128,1,fp8,fp8,0,0.14965867002805075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,1,128,1,float16,fp8,0,0.15528000394503275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,1,128,1,fp8,fp8,0,0.13782399892807007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,2,128,1,float16,fp8,0,0.155349334081014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,2,128,1,float16,float16,0,0.159578671058019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,4,128,1,float16,float16,0,0.16030933459599814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,4,128,1,float16,fp8,0,0.15615466237068176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,4,128,1,fp8,fp8,0,0.1346560021241506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,2,128,1,fp8,fp8,0,0.1336799959341685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,8,128,1,float16,fp8,0,0.1627786656220754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,8,128,1,fp8,fp8,0,0.13749333222707114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,32,8,128,1,float16,float16,0,0.16261866688728333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,32,128,1,float16,float16,0,0.10220799843470256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,1,128,1,float16,float16,0,0.09418666362762451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,1,128,1,fp8,fp8,0,0.08078399797280629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,32,128,1,float16,fp8,0,0.09872000416119893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,2,128,1,float16,float16,0,0.09402133027712505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,32,128,1,fp8,fp8,0,0.09169600407282512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,2,128,1,float16,fp8,0,0.09246933460235596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,1,128,1,float16,fp8,0,0.09227733810742696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,2,128,1,fp8,fp8,0,0.08087466657161713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,4,128,1,float16,float16,0,0.0943999985853831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,4,128,1,float16,fp8,0,0.09294933080673218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,4,128,1,fp8,fp8,0,0.08222933113574982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,8,128,1,float16,float16,0,0.0959999958674113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,8,128,1,float16,fp8,0,0.09435733159383138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,32,8,128,1,fp8,fp8,0,0.08644800384839375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,1,128,1,float16,fp8,0,3.76091734568278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,1,128,1,float16,float16,0,3.7509333292643228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,1,128,1,fp8,fp8,0,3.223338762919108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,2,128,1,fp8,fp8,0,3.3476053873697915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,2,128,1,float16,fp8,0,3.886538823445638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,2,128,1,float16,float16,0,4.137743949890137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,4,128,1,float16,float16,0,3.8972800572713218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,4,128,1,float16,fp8,0,3.9461491902669272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,1,128,1,float16,float16,0,1.8904320398966472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,4,128,1,fp8,fp8,0,3.3947998682657876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,1,128,1,float16,fp8,0,1.8881120681762695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,8,128,1,float16,float16,0,4.115802764892578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,8,128,1,fp8,fp8,0,3.585066795349121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,32,8,128,1,float16,fp8,0,4.0933332443237305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,32,128,1,float16,float16,0,2.482682704925537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,32,128,1,float16,fp8,0,3.041616121927897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,32,128,1,fp8,fp8,0,2.0186559359232583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,2,128,1,float16,float16,0,1.9229119618733723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,1,128,1,fp8,fp8,0,1.6063413619995117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,2,128,1,float16,fp8,0,1.9285653432210286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,2,128,1,fp8,fp8,0,1.6708587010701497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,4,128,1,float16,fp8,0,1.9726133346557617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,4,128,1,fp8,fp8,0,1.770901362101237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,4,128,1,float16,float16,0,1.9388747215270996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,8,128,1,float16,float16,0,2.0006240208943686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,1,128,1,float16,float16,0,1.0467466513315837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,1,128,1,float16,fp8,0,0.9586293697357178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,8,128,1,float16,fp8,0,2.0252745946248374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,1,128,1,fp8,fp8,0,0.802570660909017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,32,128,1,float16,fp8,0,1.1884533564249675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,32,128,1,float16,float16,0,1.236458698908488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,2,128,1,float16,float16,0,1.0544319947560628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,2,128,1,float16,fp8,0,0.974671999613444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,32,8,128,1,fp8,fp8,0,1.7725013097127278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,2,128,1,fp8,fp8,0,0.8363946278889974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,4,128,1,float16,float16,0,0.9773013591766357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,32,128,1,fp8,fp8,0,1.0175999800364177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,4,128,1,float16,fp8,0,0.9854773680369059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,4,128,1,fp8,fp8,0,0.857589324315389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,1,128,1,float16,float16,0,0.49198933442433673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,8,128,1,fp8,fp8,0,0.8915146986643473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,8,128,1,float16,float16,0,1.1614773273468018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,32,8,128,1,float16,fp8,0,1.0015093485514324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,1,128,1,float16,fp8,0,0.4931146701176961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,32,128,1,float16,float16,0,0.6220959822336832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,1,128,1,fp8,fp8,0,0.41171733538309735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,32,128,1,float16,fp8,0,0.5981333255767822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,2,128,1,float16,float16,0,0.4965706666310628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,32,128,1,fp8,fp8,0,0.5197386741638184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,2,128,1,fp8,fp8,0,0.42637332280476886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,2,128,1,float16,fp8,0,0.5392320156097412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,4,128,1,float16,float16,0,0.5024053255716959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,4,128,1,float16,fp8,0,0.5040586789449056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,4,128,1,fp8,fp8,0,0.4363466501235962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,8,128,1,float16,float16,0,0.514954686164856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,8,128,1,float16,fp8,0,0.5139733155568441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,32,8,128,1,fp8,fp8,0,0.4564106861750285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,32,128,1,float16,fp8,0,0.3991039991378784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,32,128,1,float16,float16,0,0.32316267490386963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,1,128,1,float16,float16,0,0.26069867610931396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,1,128,1,float16,fp8,0,0.259935994942983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,1,128,1,fp8,fp8,0,0.21707733472188315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,32,128,1,fp8,fp8,0,0.2649173339207967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,2,128,1,float16,fp8,0,0.26264532407124835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,2,128,1,fp8,fp8,0,0.23292799790700278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,2,128,1,float16,float16,0,0.2619306643803914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,4,128,1,float16,fp8,0,0.2651946743329366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,4,128,1,float16,float16,0,0.26472532749176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,4,128,1,fp8,fp8,0,0.22540799776713052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,1,128,1,float16,float16,0,0.141594668229421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,32,128,1,float16,float16,0,0.22215465704600015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,8,128,1,fp8,fp8,0,0.23481067021687826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,8,128,1,float16,float16,0,0.2716053326924642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,1,128,1,float16,fp8,0,0.1423413356145223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,32,128,1,float16,fp8,0,0.1683839956919352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,32,8,128,1,float16,fp8,0,0.27027199665705365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,32,128,1,fp8,fp8,0,0.1388053297996521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,1,128,1,fp8,fp8,0,0.11012799541155498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,2,128,1,float16,fp8,0,0.1436906655629476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,2,128,1,float16,float16,0,0.15215999881426492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,2,128,1,fp8,fp8,0,0.11285866300264995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,4,128,1,float16,float16,0,0.14471466342608133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,4,128,1,float16,fp8,0,0.14517333110173544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,8,128,1,float16,float16,0,0.14943466583887735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,4,128,1,fp8,fp8,0,0.11442666252454121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,1,128,1,float16,fp8,0,0.07668266693751018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,8,128,1,fp8,fp8,0,0.11713600158691406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,32,8,128,1,float16,fp8,0,0.14845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,32,128,1,float16,float16,0,0.1227839986483256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,32,128,1,fp8,fp8,0,0.07725333174069722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,1,128,1,float16,float16,0,0.07618133227030437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,32,128,1,float16,fp8,0,0.09259733557701111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,2,128,1,float16,fp8,0,0.07630399862925212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,2,128,1,float16,float16,0,0.07638399799664815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,4,128,1,float16,float16,0,0.08011200030644734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,4,128,1,float16,fp8,0,0.07806399961312611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,1,128,1,fp8,fp8,0,0.0613013356924057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,2,128,1,fp8,fp8,0,0.06190933287143707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,4,128,1,fp8,fp8,0,0.06408533453941345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,8,128,1,float16,float16,0,0.0795413355032603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,8,128,1,float16,fp8,0,0.07896533111731212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,32,128,1,float16,float16,0,0.05340800185998281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,32,8,128,1,fp8,fp8,0,0.067221333583196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,1,128,1,float16,float16,0,0.043247997760772705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,32,128,1,float16,fp8,0,0.048672000567118325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,1,128,1,float16,fp8,0,0.04301866888999939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,1,128,1,fp8,fp8,0,0.03602666656176249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,32,128,1,fp8,fp8,0,0.0450133333603541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,2,128,1,float16,float16,0,0.04313066601753235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,2,128,1,float16,fp8,0,0.04351999859015147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,2,128,1,fp8,fp8,0,0.03604800005753835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,4,128,1,float16,fp8,0,0.04362666606903076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,4,128,1,float16,float16,0,0.04365866879622141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,4,128,1,fp8,fp8,0,0.036746665835380554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,8,128,1,float16,float16,0,0.04409066836039225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,8,128,1,fp8,fp8,0,0.03774933268626531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,32,8,128,1,float16,fp8,0,0.04426133135954539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,1,128,1,float16,float16,0,3.296741485595703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,1,128,1,float16,fp8,0,2.9218454360961914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,1,128,1,fp8,fp8,0,2.7225707372029624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,2,128,1,fp8,fp8,0,2.810175895690918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,2,128,1,float16,fp8,0,3.026752154032389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,2,128,1,float16,float16,0,3.03765869140625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,4,128,1,float16,fp8,0,3.118506749471029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,4,128,1,float16,float16,0,3.0882345835367837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,1,128,1,float16,float16,0,1.4721760749816895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,4,128,1,fp8,fp8,0,3.029621442159017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,8,128,1,float16,fp8,0,3.2582880655924478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,8,128,1,float16,float16,0,3.239434560139974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,32,8,128,1,fp8,fp8,0,3.010474522908529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,32,128,1,float16,fp8,0,1.9923733075459797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,1,128,1,float16,fp8,0,1.473375956217448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,32,128,1,float16,float16,0,2.070474624633789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,1,128,1,fp8,fp8,0,1.3643733660380046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,32,128,1,fp8,fp8,0,1.7563947041829426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,2,128,1,float16,float16,0,1.5121280352274578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,2,128,1,float16,fp8,0,1.5057493845621746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,2,128,1,fp8,fp8,0,1.3814187049865723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,4,128,1,float16,float16,0,1.5235679944356282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,4,128,1,float16,fp8,0,1.5376799901326497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,4,128,1,fp8,fp8,0,1.433359940846761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,8,128,1,float16,float16,0,1.9108853340148926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,1,128,1,float16,float16,0,0.7488266626993815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,1,128,1,float16,fp8,0,0.7474400202433268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,8,128,1,float16,fp8,0,1.6018187204996746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,1,128,1,fp8,fp8,0,0.6941226323445638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,32,8,128,1,fp8,fp8,0,1.5151039759318035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,32,128,1,float16,float16,0,1.0452906290690105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,2,128,1,float16,float16,0,0.7612799803415934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,32,128,1,float16,fp8,0,0.9987040360768636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,32,128,1,fp8,fp8,0,0.8877333005269369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,2,128,1,float16,fp8,0,0.7627893288930258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,2,128,1,fp8,fp8,0,0.6938827037811279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,4,128,1,float16,float16,0,0.8887946605682373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,4,128,1,float16,fp8,0,0.7731946309407552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,4,128,1,fp8,fp8,0,0.7215786774953207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,8,128,1,float16,float16,0,0.7963306903839111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,8,128,1,float16,fp8,0,0.789568026860555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,1,128,1,float16,float16,0,0.3842293421427409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,32,8,128,1,fp8,fp8,0,0.7549333572387695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,1,128,1,float16,fp8,0,0.38499200344085693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,2,128,1,float16,float16,0,0.38945066928863525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,32,128,1,float16,float16,0,0.5262399911880493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,32,128,1,float16,fp8,0,0.503765344619751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,1,128,1,fp8,fp8,0,0.339685320854187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,2,128,1,float16,fp8,0,0.39085865020751953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,32,128,1,fp8,fp8,0,0.5756959915161133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,2,128,1,fp8,fp8,0,0.3550026814142863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,4,128,1,float16,float16,0,0.3949226538340251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,4,128,1,float16,fp8,0,0.39608001708984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,4,128,1,fp8,fp8,0,0.3611733516057332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,8,128,1,float16,fp8,0,0.4053066571553548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,8,128,1,float16,float16,0,0.40743998686472577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,1,128,1,float16,float16,0,0.20352532466252646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,32,128,1,float16,float16,0,0.38817067941029865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,32,128,1,float16,fp8,0,0.2624693314234416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,32,8,128,1,fp8,fp8,0,0.3877919912338257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,1,128,1,fp8,fp8,0,0.18014933665593466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,1,128,1,float16,fp8,0,0.20330133040746054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,32,128,1,fp8,fp8,0,0.22990399599075317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,2,128,1,float16,fp8,0,0.20567466815312704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,2,128,1,float16,float16,0,0.2048106590906779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,2,128,1,fp8,fp8,0,0.19500267505645752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,4,128,1,float16,float16,0,0.20835200945536295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,4,128,1,float16,fp8,0,0.2077066699663798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,4,128,1,fp8,fp8,0,0.18798933426539102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,8,128,1,float16,float16,0,0.2148373325665792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,8,128,1,float16,fp8,0,0.2143253286679586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,1,128,1,float16,float16,0,0.11036266883214314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,1,128,1,float16,fp8,0,0.11016000310579936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,32,8,128,1,fp8,fp8,0,0.19947733481725058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,32,128,1,float16,fp8,0,0.1957599918047587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,32,128,1,fp8,fp8,0,0.1250933309396108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,32,128,1,float16,float16,0,0.14714133739471436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,2,128,1,float16,float16,0,0.11154666543006897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,1,128,1,fp8,fp8,0,0.09259733557701111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,2,128,1,float16,fp8,0,0.11197866996129353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,2,128,1,fp8,fp8,0,0.09517866373062134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,4,128,1,float16,fp8,0,0.12688000003496805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,4,128,1,float16,float16,0,0.11375466982523601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,4,128,1,fp8,fp8,0,0.0974826713403066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,8,128,1,float16,fp8,0,0.11763733625411987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,8,128,1,fp8,fp8,0,0.10115200281143188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,32,8,128,1,float16,float16,0,0.11838400363922119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,1,128,1,float16,float16,0,0.06377066671848297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,32,128,1,float16,float16,0,0.08589866757392883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,32,128,1,float16,fp8,0,0.08218133449554443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,32,128,1,fp8,fp8,0,0.08654933174451192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,1,128,1,float16,fp8,0,0.06420266628265381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,1,128,1,fp8,fp8,0,0.05276800195376078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,2,128,1,float16,float16,0,0.06449600060780843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,2,128,1,fp8,fp8,0,0.05369600156943003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,4,128,1,float16,fp8,0,0.0650079995393753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,4,128,1,float16,float16,0,0.06526933113733928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,4,128,1,fp8,fp8,0,0.058176000912984215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,2,128,1,float16,fp8,0,0.06411199768384297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,8,128,1,float16,float16,0,0.06705600023269653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,8,128,1,float16,fp8,0,0.06669333577156067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,1,128,1,float16,float16,0,0.03783999880154928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,32,128,1,float16,fp8,0,0.04299733539422353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,32,128,1,float16,float16,0,0.044122666120529175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,32,128,1,fp8,fp8,0,0.03995199998219808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,1,128,1,fp8,fp8,0,0.031717332700888314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,32,8,128,1,fp8,fp8,0,0.05871466795603434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,2,128,1,float16,float16,0,0.03755733370780945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,1,128,1,float16,fp8,0,0.03748266647259394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,2,128,1,float16,fp8,0,0.037685332198937736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,2,128,1,fp8,fp8,0,0.03175999969244003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,4,128,1,float16,fp8,0,0.037765334049860634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,4,128,1,fp8,fp8,0,0.032602667808532715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,4,128,1,float16,float16,0,0.037808001041412354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,8,128,1,float16,fp8,0,0.0383840004603068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,8,128,1,float16,float16,0,0.038373333712418876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,32,128,1,float16,float16,0,0.027104000250498455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,32,8,128,1,fp8,fp8,0,0.03382933388153712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,32,128,1,float16,fp8,0,0.02720000098148982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,1,128,1,float16,float16,0,0.025498665869235992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,32,128,1,fp8,fp8,0,0.024773334463437397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,1,128,1,fp8,fp8,0,0.02327466756105423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,1,128,1,float16,fp8,0,0.02515733242034912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,2,128,1,float16,fp8,0,0.025616000096003216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,4,128,1,float16,float16,0,0.026399999856948853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,4,128,1,float16,fp8,0,0.02647999922434489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,2,128,1,float16,float16,0,0.02555199960867564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,4,128,1,fp8,fp8,0,0.02388266722361247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,8,128,1,float16,float16,0,0.026522666215896606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,2,128,1,fp8,fp8,0,0.02306666721900304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,8,128,1,float16,fp8,0,0.026704000929991405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,32,8,128,1,fp8,fp8,0,0.024559999505678814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,1,128,1,float16,float16,0,1.2770826816558838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,1,128,1,float16,fp8,0,1.2765599886576335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,2,128,1,float16,float16,0,1.303536017735799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,2,128,1,float16,fp8,0,1.3125920295715332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,2,128,1,fp8,fp8,0,1.1409119764963787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,4,128,1,float16,float16,0,1.333456039428711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,1,128,1,fp8,fp8,0,1.1103413105010986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,4,128,1,float16,fp8,0,1.5774399439493816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,1,128,1,float16,float16,0,0.7531786759694418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,4,128,1,fp8,fp8,0,1.2161973317464192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,8,128,1,float16,fp8,0,1.3953812917073567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,8,128,1,fp8,fp8,0,1.3355147043863933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,1,128,1,float16,fp8,0,0.7518560091654459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,32,8,128,1,float16,float16,0,1.3782986005147297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,32,128,1,float16,fp8,0,0.9254026412963867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,32,128,1,float16,float16,0,0.9704053401947021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,32,128,1,fp8,fp8,0,0.7867999871571859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,1,128,1,fp8,fp8,0,0.5495253403981527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,2,128,1,float16,float16,0,0.6602720022201538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,2,128,1,float16,fp8,0,0.6623626550038656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,4,128,1,float16,float16,0,0.6696533362070719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,2,128,1,fp8,fp8,0,0.6382026672363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,4,128,1,float16,fp8,0,0.673205296198527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,4,128,1,fp8,fp8,0,0.6140693426132202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,8,128,1,float16,float16,0,0.6951253414154053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,1,128,1,float16,float16,0,0.3349866469701131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,1,128,1,float16,fp8,0,0.33536001046498615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,8,128,1,float16,fp8,0,0.6905653476715088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,32,8,128,1,fp8,fp8,0,0.6658666531244913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,32,128,1,float16,fp8,0,0.4678293466567993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,1,128,1,fp8,fp8,0,0.2802880009015401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,2,128,1,float16,float16,0,0.3391520182291667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,32,128,1,fp8,fp8,0,0.561189333597819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,2,128,1,fp8,fp8,0,0.2946293354034424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,32,128,1,float16,float16,0,0.49428268273671466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,2,128,1,float16,fp8,0,0.34016533692677814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,4,128,1,float16,float16,0,0.34381333986918133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,4,128,1,float16,fp8,0,0.34407468636830646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,4,128,1,fp8,fp8,0,0.3097226619720459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,8,128,1,float16,float16,0,0.356170654296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,8,128,1,fp8,fp8,0,0.3376693328221639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,32,8,128,1,float16,fp8,0,0.44843733310699463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,32,128,1,float16,float16,0,0.2541760007540385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,1,128,1,float16,float16,0,0.17715734243392944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,1,128,1,fp8,fp8,0,0.15079999963442484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,32,128,1,float16,fp8,0,0.24478934208552042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,1,128,1,float16,fp8,0,0.17760000626246134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,32,128,1,fp8,fp8,0,0.20672533909479776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,2,128,1,float16,fp8,0,0.20519999663035074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,2,128,1,float16,float16,0,0.17828265825907388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,2,128,1,fp8,fp8,0,0.15780267119407654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,4,128,1,float16,float16,0,0.18195732434590658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,4,128,1,fp8,fp8,0,0.1613599956035614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,4,128,1,float16,fp8,0,0.1818986733754476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,8,128,1,float16,float16,0,0.18902933597564697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,32,128,1,float16,fp8,0,0.1327946682771047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,8,128,1,float16,fp8,0,0.18625599145889282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,32,128,1,float16,float16,0,0.137855996688207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,32,8,128,1,fp8,fp8,0,0.2013173302014669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,32,128,1,fp8,fp8,0,0.11795733372370402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,1,128,1,float16,float16,0,0.09614400068918864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,1,128,1,float16,fp8,0,0.09659733374913533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,1,128,1,fp8,fp8,0,0.08355733752250671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,2,128,1,float16,float16,0,0.09788800279299419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,2,128,1,float16,fp8,0,0.09717866778373718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,4,128,1,float16,float16,0,0.11566932996114095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,2,128,1,fp8,fp8,0,0.08633599678675334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,4,128,1,float16,fp8,0,0.09990933537483215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,8,128,1,float16,float16,0,0.10389332969983418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,4,128,1,fp8,fp8,0,0.08854933579762776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,8,128,1,fp8,fp8,0,0.0925600032011668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,32,8,128,1,float16,fp8,0,0.10276800394058228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,1,128,1,float16,float16,0,0.05709866682688395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,32,128,1,float16,float16,0,0.08544533451398213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,32,128,1,float16,fp8,0,0.10852799812952678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,32,128,1,fp8,fp8,0,0.06487466891606648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,1,128,1,float16,fp8,0,0.057376002271970115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,2,128,1,float16,float16,0,0.05753066639105479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,2,128,1,float16,fp8,0,0.057706668972969055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,4,128,1,float16,float16,0,0.05789333085219065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,1,128,1,fp8,fp8,0,0.04814399778842926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,4,128,1,float16,fp8,0,0.05952533086140951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,2,128,1,fp8,fp8,0,0.04872000217437744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,8,128,1,float16,float16,0,0.06029866635799408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,4,128,1,fp8,fp8,0,0.050255998969078064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,8,128,1,float16,fp8,0,0.06033066908518473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,32,128,1,float16,float16,0,0.04115733255942663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,32,8,128,1,fp8,fp8,0,0.053039997816085815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,32,128,1,fp8,fp8,0,0.043653334180514015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,1,128,1,float16,float16,0,0.034714666505654655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,1,128,1,float16,fp8,0,0.03473600000143051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,32,128,1,float16,fp8,0,0.0397119993964831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,2,128,1,float16,fp8,0,0.03483733286460241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,2,128,1,float16,float16,0,0.034858666360378265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,4,128,1,float16,float16,0,0.035002666215101876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,2,128,1,fp8,fp8,0,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,1,128,1,fp8,fp8,0,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,4,128,1,fp8,fp8,0,0.03046400099992752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,4,128,1,float16,fp8,0,0.03501333296298981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,8,128,1,float16,fp8,0,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,8,128,1,fp8,fp8,0,0.031744000812371574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,32,128,1,float16,float16,0,0.02497600018978119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,32,128,1,float16,fp8,0,0.025013332565625507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,1,128,1,float16,float16,0,0.023285334308942158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,1,128,1,float16,fp8,0,0.0232640008131663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,32,8,128,1,float16,float16,0,0.03570666660865148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,1,128,1,fp8,fp8,0,0.02075733368595441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,32,128,1,fp8,fp8,0,0.022885332504908245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,2,128,1,float16,fp8,0,0.02316266546646754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,2,128,1,float16,float16,0,0.02329600105683009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,4,128,1,float16,float16,0,0.023733332753181458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,2,128,1,fp8,fp8,0,0.020768000433842342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,4,128,1,float16,fp8,0,0.023797333240509033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,8,128,1,float16,float16,0,0.02364266663789749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,8,128,1,fp8,fp8,0,0.022405333817005157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,8,128,1,float16,fp8,0,0.02390933285156886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,32,128,1,float16,float16,0,0.02045866722861926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,32,4,128,1,fp8,fp8,0,0.021770666042963665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,32,128,1,float16,fp8,0,0.020576000213623047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,1,128,1,float16,float16,0,0.01924266666173935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,32,128,1,fp8,fp8,0,0.019226666539907455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,1,128,1,float16,fp8,0,0.019013332823912304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,1,128,1,fp8,fp8,0,0.018330667167901993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,2,128,1,float16,float16,0,0.054469332098960876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,2,128,1,float16,fp8,0,0.019381333142518997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,4,128,1,float16,float16,0,0.01926400015751521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,4,128,1,float16,fp8,0,0.019706666469573975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,8,128,1,float16,float16,0,0.01937066639463107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,4,128,1,fp8,fp8,0,0.01886933296918869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,8,128,1,float16,fp8,0,0.01995733380317688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,2,128,1,fp8,fp8,0,0.018394666413466137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,32,8,128,1,fp8,fp8,0,0.021664001047611237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,1,128,1,float16,float16,0,0.6063786745071411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,1,128,1,float16,fp8,0,0.6058826843897501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,2,128,1,float16,fp8,0,0.621120015780131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,2,128,1,float16,float16,0,0.6164693435033163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,1,128,1,fp8,fp8,0,0.5449706713358561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,2,128,1,fp8,fp8,0,0.6549599965413412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,4,128,1,float16,float16,0,0.6275306542714437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,4,128,1,float16,fp8,0,0.6277493238449097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,1,128,1,float16,float16,0,0.3129706581433614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,4,128,1,fp8,fp8,0,0.6245226860046387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,8,128,1,float16,float16,0,0.6525493462880453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,8,128,1,fp8,fp8,0,0.6667359670003256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,32,8,128,1,float16,fp8,0,0.645962675412496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,32,128,1,float16,float16,0,0.5092959801355997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,32,128,1,float16,fp8,0,0.7143572966257731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,1,128,1,float16,fp8,0,0.3123786648114522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,1,128,1,fp8,fp8,0,0.2805866599082947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,32,128,1,fp8,fp8,0,0.4018933375676473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,2,128,1,float16,fp8,0,0.31885333855946857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,2,128,1,float16,float16,0,0.31946667035420734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,2,128,1,fp8,fp8,0,0.2951359947522481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,4,128,1,float16,float16,0,0.32478400071461994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,4,128,1,float16,fp8,0,0.40805331865946454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,4,128,1,fp8,fp8,0,0.30847465991973877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,8,128,1,float16,float16,0,0.3383893171946208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,1,128,1,float16,float16,0,0.19655466079711914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,8,128,1,float16,fp8,0,0.336298664410909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,1,128,1,float16,fp8,0,0.16871466239293417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,32,8,128,1,fp8,fp8,0,0.34071465333302814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,1,128,1,fp8,fp8,0,0.1508799990018209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,32,128,1,float16,float16,0,0.26502933104832965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,32,128,1,fp8,fp8,0,0.20523732900619507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,32,128,1,float16,fp8,0,0.2549920082092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,2,128,1,float16,float16,0,0.20229333639144897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,2,128,1,fp8,fp8,0,0.15728533267974854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,2,128,1,float16,fp8,0,0.17189866304397583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,4,128,1,float16,float16,0,0.1747466723124186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,4,128,1,float16,fp8,0,0.17453332742055258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,4,128,1,fp8,fp8,0,0.16170133153597513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,8,128,1,float16,float16,0,0.18222399552663168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,8,128,1,float16,fp8,0,0.1804479956626892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,32,8,128,1,fp8,fp8,0,0.2007840077082316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,1,128,1,float16,float16,0,0.09388800462086995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,32,128,1,fp8,fp8,0,0.11226133505503337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,1,128,1,float16,fp8,0,0.09321600198745728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,1,128,1,fp8,fp8,0,0.08365866541862488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,2,128,1,float16,fp8,0,0.1062506635983785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,2,128,1,float16,float16,0,0.09451199571291606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,32,128,1,float16,float16,0,0.1441439986228943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,32,128,1,float16,fp8,0,0.13913599650065103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,2,128,1,fp8,fp8,0,0.08601599931716919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,4,128,1,float16,fp8,0,0.09649599591890971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,4,128,1,float16,float16,0,0.09656000137329102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,8,128,1,float16,float16,0,0.12682666381200156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,4,128,1,fp8,fp8,0,0.08731200297673543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,8,128,1,float16,fp8,0,0.10117333134015401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,32,8,128,1,fp8,fp8,0,0.09105066458384196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,32,128,1,float16,fp8,0,0.07771199941635132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,1,128,1,float16,float16,0,0.05426133175690969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,1,128,1,float16,fp8,0,0.05500266452630361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,32,128,1,float16,float16,0,0.08108266691366832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,1,128,1,fp8,fp8,0,0.04790399968624115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,32,128,1,fp8,fp8,0,0.06542400022347768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,2,128,1,float16,float16,0,0.05382933219273885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,2,128,1,fp8,fp8,0,0.04884799818197886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,2,128,1,float16,fp8,0,0.05380799869696299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,4,128,1,float16,float16,0,0.055029332637786865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,4,128,1,float16,fp8,0,0.05726933479309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,4,128,1,fp8,fp8,0,0.05056533217430115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,8,128,1,float16,fp8,0,0.05780800183614095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,8,128,1,float16,float16,0,0.057717333237330117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,32,8,128,1,fp8,fp8,0,0.05259199937184652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,32,128,1,float16,float16,0,0.040778666734695435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,32,128,1,float16,fp8,0,0.03886399914820989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,1,128,1,float16,float16,0,0.032986665765444435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,32,128,1,fp8,fp8,0,0.037263999382654824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,1,128,1,float16,fp8,0,0.033301333586374916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,2,128,1,float16,float16,0,0.03314133236805598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,1,128,1,fp8,fp8,0,0.02958933264017105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,2,128,1,float16,fp8,0,0.03302400062481562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,2,128,1,fp8,fp8,0,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,4,128,1,float16,float16,0,0.03324266771475474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,4,128,1,float16,fp8,0,0.033413333197434746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,4,128,1,fp8,fp8,0,0.030346666773160298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,8,128,1,float16,fp8,0,0.034517332911491394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,32,128,1,float16,float16,0,0.024058667321999867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,8,128,1,fp8,fp8,0,0.031471999982992806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,32,128,1,float16,fp8,0,0.023989332218964893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,1,128,1,float16,fp8,0,0.02252800017595291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,32,128,1,fp8,fp8,0,0.023269332945346832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,32,8,128,1,float16,float16,0,0.03432533393303553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,1,128,1,float16,float16,0,0.022309333086013794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,1,128,1,fp8,fp8,0,0.02110933264096578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,2,128,1,float16,float16,0,0.02199999988079071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,2,128,1,float16,fp8,0,0.022602667411168415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,4,128,1,float16,float16,0,0.022709332406520844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,8,128,1,float16,float16,0,0.02292266736427943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,4,128,1,float16,fp8,0,0.022895999252796173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,2,128,1,fp8,fp8,0,0.02107200026512146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,8,128,1,float16,fp8,0,0.023376000424226124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,4,128,1,fp8,fp8,0,0.021920000513394673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,1,128,1,float16,float16,0,0.017957333475351334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,32,128,1,float16,float16,0,0.01937599976857503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,32,128,1,float16,fp8,0,0.018629333625237148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,32,8,128,1,fp8,fp8,0,0.022613334159056347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,1,128,1,float16,fp8,0,0.018874666343132656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,32,128,1,fp8,fp8,0,0.019280000279347103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,1,128,1,fp8,fp8,0,0.018357332795858383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,2,128,1,float16,fp8,0,0.01815466706951459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,2,128,1,fp8,fp8,0,0.01828266680240631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,4,128,1,float16,float16,0,0.01802666609485944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,4,128,1,float16,fp8,0,0.01846933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,8,128,1,float16,float16,0,0.018272000054518383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,4,128,1,fp8,fp8,0,0.019130667050679524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,2,128,1,float16,float16,0,0.01844266677896182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,8,128,1,float16,fp8,0,0.01865600049495697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,32,8,128,1,fp8,fp8,0,0.019194666296243668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,32,128,1,float16,float16,0,0.017050666113694508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,32,128,1,float16,fp8,0,0.017050666113694508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,1,128,1,float16,float16,0,0.016864000509182613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,32,128,1,fp8,fp8,0,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,1,128,1,float16,fp8,0,0.01718933383623759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,1,128,1,fp8,fp8,0,0.017055999487638474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,2,128,1,float16,float16,0,0.016906666258970898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,2,128,1,float16,fp8,0,0.017162666966517765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,4,128,1,float16,float16,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,2,128,1,fp8,fp8,0,0.017375999440749485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,4,128,1,float16,fp8,0,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,4,128,1,fp8,fp8,0,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,8,128,1,float16,float16,0,0.016965333372354507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,8,128,1,fp8,fp8,0,0.01764800027012825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,32,8,128,1,float16,fp8,0,0.017445333302021027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,1,128,1,float16,fp8,0,0.37345067660013836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,1,128,1,fp8,fp8,0,0.38273600737253827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,2,128,1,float16,float16,0,0.4532586733500163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,2,128,1,fp8,fp8,0,0.3946133454640706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,2,128,1,float16,fp8,0,0.3797279993693034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,1,128,1,float16,float16,0,0.3742026487986247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,4,128,1,float16,fp8,0,0.3862186670303345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,4,128,1,float16,float16,0,0.3854293425877889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,1,128,1,float16,float16,0,0.19682133197784424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,4,128,1,fp8,fp8,0,0.40425066153208417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,8,128,1,fp8,fp8,0,0.43175466855367023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,8,128,1,float16,fp8,0,0.5229440132776896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,32,8,128,1,float16,float16,0,0.4002079963684082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,32,128,1,float16,float16,0,0.26109333833058673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,32,128,1,fp8,fp8,0,0.2537440061569214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,32,128,1,float16,fp8,0,0.24964799483617148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,1,128,1,fp8,fp8,0,0.20225600401560465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,1,128,1,float16,fp8,0,0.19706666469573975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,2,128,1,float16,float16,0,0.19952533642450967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,2,128,1,float16,fp8,0,0.23244800170262656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,4,128,1,float16,float16,0,0.20266133546829224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,2,128,1,fp8,fp8,0,0.20755734046300253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,8,128,1,float16,float16,0,0.2100480000178019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,4,128,1,float16,fp8,0,0.20175999402999878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,8,128,1,float16,fp8,0,0.2087679902712504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,4,128,1,fp8,fp8,0,0.2116373380025228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,32,128,1,float16,float16,0,0.20176533857981363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,32,8,128,1,fp8,fp8,0,0.21996800104777017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,1,128,1,float16,float16,0,0.10520000259081523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,32,128,1,float16,fp8,0,0.13504532972971597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,32,128,1,fp8,fp8,0,0.1350826621055603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,1,128,1,fp8,fp8,0,0.1111199955145518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,1,128,1,float16,fp8,0,0.10523733496665955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,2,128,1,float16,fp8,0,0.12082133690516154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,2,128,1,fp8,fp8,0,0.11275733510653178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,2,128,1,float16,float16,0,0.10630933443705241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,4,128,1,float16,float16,0,0.10879466931025188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,4,128,1,float16,fp8,0,0.10900266965230306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,4,128,1,fp8,fp8,0,0.11476799845695496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,8,128,1,float16,float16,0,0.11442133784294128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,8,128,1,float16,fp8,0,0.11389866471290588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,32,8,128,1,fp8,fp8,0,0.12691199779510498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,32,128,1,float16,float16,0,0.07866133252779643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,1,128,1,float16,float16,0,0.05964266757170359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,32,128,1,fp8,fp8,0,0.07694933315118153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,1,128,1,float16,fp8,0,0.05973866581916809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,1,128,1,fp8,fp8,0,0.061664000153541565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,32,128,1,float16,fp8,0,0.07534400125344594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,2,128,1,float16,fp8,0,0.060378665725390114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,2,128,1,float16,float16,0,0.05937066674232483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,2,128,1,fp8,fp8,0,0.06285866598288219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,4,128,1,float16,fp8,0,0.06073066592216492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,4,128,1,float16,float16,0,0.06035199761390686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,4,128,1,fp8,fp8,0,0.06477333108584087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,8,128,1,float16,float16,0,0.06226666768391927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,8,128,1,float16,fp8,0,0.06204266846179962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,32,8,128,1,fp8,fp8,0,0.07102933526039124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,32,128,1,float16,fp8,0,0.03857066730658213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,1,128,1,float16,float16,0,0.035743998984495796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,32,128,1,fp8,fp8,0,0.04433600107828776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,32,128,1,float16,float16,0,0.0399893323580424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,1,128,1,float16,fp8,0,0.035232000052928925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,1,128,1,fp8,fp8,0,0.03658133248488108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,2,128,1,float16,fp8,0,0.03575466573238373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,2,128,1,float16,float16,0,0.03530666728814443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,2,128,1,fp8,fp8,0,0.036687999963760376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,4,128,1,float16,fp8,0,0.03583999971548716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,4,128,1,float16,float16,0,0.03579200059175491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,8,128,1,float16,float16,0,0.03626666714747747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,8,128,1,float16,fp8,0,0.03682666768630346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,4,128,1,fp8,fp8,0,0.037290667494138084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,32,8,128,1,fp8,fp8,0,0.03835200021664301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,32,128,1,float16,float16,0,0.024864000578721363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,32,128,1,fp8,fp8,0,0.02622933437426885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,32,128,1,float16,fp8,0,0.02475733309984207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,1,128,1,float16,float16,0,0.023472001155217487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,1,128,1,float16,fp8,0,0.023498666783173878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,1,128,1,fp8,fp8,0,0.024688000480333965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,2,128,1,float16,fp8,0,0.023904000719388325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,2,128,1,fp8,fp8,0,0.024325333535671234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,4,128,1,float16,fp8,0,0.024432001014550526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,4,128,1,fp8,fp8,0,0.0249439999461174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,2,128,1,float16,float16,0,0.02351466566324234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,4,128,1,float16,float16,0,0.024005333582560223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,8,128,1,float16,float16,0,0.023792001108328503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,8,128,1,float16,fp8,0,0.024383999407291412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,32,8,128,1,fp8,fp8,0,0.0264533335963885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,32,128,1,float16,float16,0,0.01893866683046023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,32,128,1,float16,fp8,0,0.018885333091020584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,1,128,1,float16,fp8,0,0.017765333255132038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,32,128,1,fp8,fp8,0,0.019493332753578823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,1,128,1,fp8,fp8,0,0.018229333062966663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,2,128,1,float16,float16,0,0.017914666483799618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,2,128,1,float16,fp8,0,0.017898666361967724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,1,128,1,float16,float16,0,0.01762666677435239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,4,128,1,float16,fp8,0,0.017909333109855652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,4,128,1,float16,float16,0,0.01810666670401891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,2,128,1,fp8,fp8,0,0.018543999642133713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,4,128,1,fp8,fp8,0,0.019248000035683315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,8,128,1,float16,float16,0,0.018325333793958027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,8,128,1,fp8,fp8,0,0.019744000087181728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,32,8,128,1,float16,fp8,0,0.018346666047970455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,2,128,1,float16,float16,0,0.01569066693385442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,32,128,1,float16,fp8,0,0.016208000481128693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,32,128,1,fp8,fp8,0,0.017594666530688603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,1,128,1,float16,fp8,0,0.016250666230916977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,32,128,1,float16,float16,0,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,1,128,1,fp8,fp8,0,0.01695466662446658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,1,128,1,float16,float16,0,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,2,128,1,float16,fp8,0,0.0161920003592968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,2,128,1,fp8,fp8,0,0.017125333348910015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,4,128,1,float16,float16,0,0.015642666568358738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,4,128,1,float16,fp8,0,0.016229332735141117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,4,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,8,128,1,float16,fp8,0,0.0163680004576842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,32,128,1,float16,float16,0,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,8,128,1,fp8,fp8,0,0.017765333255132038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,32,128,1,float16,fp8,0,0.01563199982047081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,32,8,128,1,float16,float16,0,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,32,128,1,fp8,fp8,0,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,1,128,1,float16,float16,0,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,1,128,1,float16,fp8,0,0.015781333049138386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,1,128,1,fp8,fp8,0,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,2,128,1,fp8,fp8,0,0.01672533278663953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,2,128,1,float16,float16,0,0.015263999501864115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,2,128,1,float16,fp8,0,0.015605332950750986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,4,128,1,float16,fp8,0,0.015743999431530636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,4,128,1,float16,float16,0,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,4,128,1,fp8,fp8,0,0.016442666451136272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,8,128,1,float16,float16,0,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,8,128,1,fp8,fp8,0,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,32,8,128,1,float16,fp8,0,0.01591466615597407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,1,128,1,float16,float16,0,0.2624533375104268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,1,128,1,fp8,fp8,0,0.30697067578633624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,1,128,1,float16,fp8,0,0.2632053295771281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,2,128,1,float16,fp8,0,0.2666933337847392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,2,128,1,fp8,fp8,0,0.31563733021418255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,4,128,1,float16,float16,0,0.307696004708608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,4,128,1,float16,fp8,0,0.2693013350168864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,2,128,1,float16,float16,0,0.2653973301251729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,4,128,1,fp8,fp8,0,0.3142026662826538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,8,128,1,float16,float16,0,0.27743999163309735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,8,128,1,float16,fp8,0,0.2759360074996948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,32,128,1,float16,float16,0,0.16847467422485352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,32,128,1,float16,fp8,0,0.21441600720087686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,32,8,128,1,fp8,fp8,0,0.32477333148320514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,1,128,1,float16,float16,0,0.13860799868901572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,32,128,1,fp8,fp8,0,0.1853440006573995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,1,128,1,float16,fp8,0,0.13807466626167297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,2,128,1,float16,float16,0,0.13912533720334372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,1,128,1,fp8,fp8,0,0.16433599591255188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,2,128,1,fp8,fp8,0,0.17315733432769775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,2,128,1,float16,fp8,0,0.13924266894658408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,4,128,1,float16,float16,0,0.1427893340587616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,4,128,1,float16,fp8,0,0.142277330160141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,4,128,1,fp8,fp8,0,0.16831467549006143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,8,128,1,float16,float16,0,0.1466506620248159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,8,128,1,float16,fp8,0,0.14482667048772177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,32,128,1,float16,float16,0,0.11806399623552959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,32,8,128,1,fp8,fp8,0,0.17141334215799967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,32,128,1,fp8,fp8,0,0.1019040048122406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,32,128,1,float16,fp8,0,0.08945600191752116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,1,128,1,float16,fp8,0,0.07526400188604991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,1,128,1,float16,float16,0,0.07462400197982788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,1,128,1,fp8,fp8,0,0.08982400099436443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,4,128,1,float16,float16,0,0.07602133353551228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,2,128,1,float16,fp8,0,0.07545066873232524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,2,128,1,float16,float16,0,0.07422400017579396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,2,128,1,fp8,fp8,0,0.09095999598503113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,4,128,1,fp8,fp8,0,0.0916373332341512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,8,128,1,float16,fp8,0,0.07787199815114339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,8,128,1,float16,float16,0,0.07735466460386912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,4,128,1,float16,fp8,0,0.07633600135644276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,32,8,128,1,fp8,fp8,0,0.09486400087674458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,32,128,1,float16,float16,0,0.051327998439470925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,1,128,1,float16,float16,0,0.04285866518815359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,1,128,1,float16,fp8,0,0.04252799848715464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,32,128,1,fp8,fp8,0,0.057909334699312844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,2,128,1,float16,float16,0,0.043231998880704246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,2,128,1,float16,fp8,0,0.04313066601753235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,32,128,1,float16,fp8,0,0.045184001326560974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,2,128,1,fp8,fp8,0,0.05046399931112925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,1,128,1,fp8,fp8,0,0.0505973349014918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,4,128,1,float16,float16,0,0.043136000633239746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,4,128,1,float16,fp8,0,0.04297600189844767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,4,128,1,fp8,fp8,0,0.051130667328834534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,8,128,1,float16,float16,0,0.044218664367993675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,8,128,1,float16,fp8,0,0.04400533437728882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,32,128,1,float16,float16,0,0.027973333994547527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,32,8,128,1,fp8,fp8,0,0.052015999952952065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,32,128,1,float16,fp8,0,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,1,128,1,float16,fp8,0,0.02717333287000656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,1,128,1,fp8,fp8,0,0.031471999982992806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,2,128,1,float16,float16,0,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,2,128,1,float16,fp8,0,0.027061333258946735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,1,128,1,float16,float16,0,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,32,128,1,fp8,fp8,0,0.03307733436425527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,4,128,1,float16,fp8,0,0.027552001178264618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,4,128,1,fp8,fp8,0,0.03209600100914637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,8,128,1,float16,float16,0,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,2,128,1,fp8,fp8,0,0.03146666785081228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,4,128,1,float16,float16,0,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,8,128,1,fp8,fp8,0,0.03286933402220408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,32,8,128,1,float16,fp8,0,0.02758399893840154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,32,128,1,float16,float16,0,0.019754666835069656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,1,128,1,float16,float16,0,0.018581333259741466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,32,128,1,float16,fp8,0,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,32,128,1,fp8,fp8,0,0.023029332359631855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,2,128,1,float16,fp8,0,0.0195573332409064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,2,128,1,float16,float16,0,0.019274666905403137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,2,128,1,fp8,fp8,0,0.021685334543387096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,1,128,1,float16,fp8,0,0.018933333456516266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,1,128,1,fp8,fp8,0,0.021989333132902782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,4,128,1,float16,float16,0,0.019359999646743137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,4,128,1,float16,fp8,0,0.019482667247454327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,8,128,1,float16,float16,0,0.019226666539907455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,4,128,1,fp8,fp8,0,0.022463999688625336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,8,128,1,float16,fp8,0,0.019551999866962433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,32,8,128,1,fp8,fp8,0,0.022986667851607006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,32,128,1,fp8,fp8,0,0.018063999712467194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,1,128,1,float16,fp8,0,0.015589332828919092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,1,128,1,float16,float16,0,0.015397333850463232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,1,128,1,fp8,fp8,0,0.017263999829689663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,32,128,1,float16,fp8,0,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,32,128,1,float16,float16,0,0.01580799991885821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,2,128,1,float16,float16,0,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,2,128,1,fp8,fp8,0,0.01782400036851565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,2,128,1,float16,fp8,0,0.01579733317097028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,4,128,1,float16,float16,0,0.015247999380032221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,4,128,1,float16,fp8,0,0.016021333634853363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,4,128,1,fp8,fp8,0,0.0180479995906353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,32,128,1,float16,float16,0,0.014650666465361914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,8,128,1,fp8,fp8,0,0.018613333503405254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,32,128,1,float16,fp8,0,0.01504533365368843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,8,128,1,float16,float16,0,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,32,128,1,fp8,fp8,0,0.01735466718673706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,1,128,1,float16,fp8,0,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,32,8,128,1,float16,fp8,0,0.015765332927306492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,1,128,1,float16,float16,0,0.014677333335081736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,1,128,1,fp8,fp8,0,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,2,128,1,float16,fp8,0,0.015114666273196539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,2,128,1,float16,float16,0,0.014677333335081736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,4,128,1,float16,float16,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,4,128,1,float16,fp8,0,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,8,128,1,float16,float16,0,0.014645333091417948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,2,128,1,fp8,fp8,0,0.016970666746298473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,8,128,1,float16,fp8,0,0.015034666905800501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,8,128,1,fp8,fp8,0,0.016773333152135212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,32,128,1,float16,float16,0,0.014368000129858652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,32,4,128,1,fp8,fp8,0,0.016869333883126576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,1,128,1,fp8,fp8,0,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,32,128,1,float16,fp8,0,0.014607999473810196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,32,128,1,fp8,fp8,0,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,1,128,1,float16,float16,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,1,128,1,float16,fp8,0,0.014864000181357065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,4,128,1,float16,float16,0,0.014127999544143677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,2,128,1,float16,float16,0,0.014250667144854864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,2,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,4,128,1,float16,fp8,0,0.014981333166360855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,4,128,1,fp8,fp8,0,0.01623999948302905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,8,128,1,float16,float16,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,8,128,1,fp8,fp8,0,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,2,128,1,fp8,fp8,0,0.016149333367745083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,32,8,128,1,float16,fp8,0,0.01488000030318896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,1,128,1,float16,float16,0,0.22325867414474487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,1,128,1,fp8,fp8,0,0.27180800835291546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,1,128,1,float16,fp8,0,0.22302399079004923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,2,128,1,float16,float16,0,0.2230186661084493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,2,128,1,float16,fp8,0,0.22334933280944824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,2,128,1,fp8,fp8,0,0.26925865809122723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,4,128,1,float16,fp8,0,0.2244373361269633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,4,128,1,float16,float16,0,0.2242400050163269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,4,128,1,fp8,fp8,0,0.2685973246892293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,8,128,1,float16,float16,0,0.2457546591758728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,8,128,1,float16,fp8,0,0.22804266214370728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,32,128,1,float16,float16,0,0.12435199817021687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,32,8,128,1,fp8,fp8,0,0.27451733748118085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,32,128,1,float16,fp8,0,0.1204800009727478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,1,128,1,float16,float16,0,0.11613333225250244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,1,128,1,fp8,fp8,0,0.14255999525388083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,1,128,1,float16,fp8,0,0.11763733625411987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,32,128,1,fp8,fp8,0,0.15436266859372458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,2,128,1,float16,float16,0,0.11802666385968526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,2,128,1,float16,fp8,0,0.11781866351763408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,2,128,1,fp8,fp8,0,0.1421333352724711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,4,128,1,float16,float16,0,0.11852266391118367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,4,128,1,float16,fp8,0,0.11786133050918579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,4,128,1,fp8,fp8,0,0.14691733320554098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,8,128,1,float16,float16,0,0.12008532881736755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,8,128,1,fp8,fp8,0,0.14470400412877402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,32,8,128,1,float16,fp8,0,0.12026133139928182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,32,128,1,float16,float16,0,0.06513066589832306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,32,128,1,fp8,fp8,0,0.08455999692281087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,32,128,1,float16,fp8,0,0.06213866670926412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,1,128,1,float16,fp8,0,0.06508799890677135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,1,128,1,float16,float16,0,0.06453333298365276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,1,128,1,fp8,fp8,0,0.07841066519419353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,2,128,1,float16,float16,0,0.06513066589832306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,2,128,1,fp8,fp8,0,0.07820799946784973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,4,128,1,float16,float16,0,0.06543466448783875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,4,128,1,float16,fp8,0,0.06539200246334076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,4,128,1,fp8,fp8,0,0.07901866734027863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,2,128,1,float16,fp8,0,0.06538666784763336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,8,128,1,float16,float16,0,0.06576000154018402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,8,128,1,float16,fp8,0,0.0661599983771642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,32,8,128,1,fp8,fp8,0,0.07970133423805237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,32,128,1,float16,float16,0,0.03659733384847641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,32,128,1,float16,fp8,0,0.03659199923276901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,32,128,1,fp8,fp8,0,0.04644800225893656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,1,128,1,float16,float16,0,0.038549333810806274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,1,128,1,float16,fp8,0,0.038480001191298165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,2,128,1,float16,fp8,0,0.03846399982770284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,2,128,1,float16,float16,0,0.03842133283615112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,1,128,1,fp8,fp8,0,0.04577599962552389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,4,128,1,float16,float16,0,0.03896533449490865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,4,128,1,fp8,fp8,0,0.04631466666857401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,4,128,1,float16,fp8,0,0.03941333293914795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,2,128,1,fp8,fp8,0,0.04613866905371348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,8,128,1,float16,fp8,0,0.039279999832312264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,32,128,1,float16,fp8,0,0.024336000283559162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,8,128,1,fp8,fp8,0,0.04659200211366018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,32,8,128,1,float16,float16,0,0.03939733405907949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,1,128,1,float16,float16,0,0.024298667907714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,32,128,1,fp8,fp8,0,0.029653333127498627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,32,128,1,float16,float16,0,0.024527999262015026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,1,128,1,float16,fp8,0,0.025045332809289295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,2,128,1,float16,float16,0,0.02458133300145467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,1,128,1,fp8,fp8,0,0.029114666084448498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,2,128,1,float16,fp8,0,0.02458133300145467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,4,128,1,float16,float16,0,0.02493866781393687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,4,128,1,float16,fp8,0,0.02508266766866048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,4,128,1,fp8,fp8,0,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,8,128,1,float16,float16,0,0.02516799916823705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,2,128,1,fp8,fp8,0,0.029146666328112285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,8,128,1,float16,fp8,0,0.02497600018978119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,32,8,128,1,fp8,fp8,0,0.03036266565322876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,32,128,1,float16,float16,0,0.018144000321626663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,32,128,1,float16,fp8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,32,128,1,fp8,fp8,0,0.021520001192887623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,2,128,1,float16,float16,0,0.01782400036851565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,1,128,1,float16,fp8,0,0.018351999421914417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,1,128,1,float16,float16,0,0.017664000391960144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,2,128,1,float16,fp8,0,0.018405333161354065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,1,128,1,fp8,fp8,0,0.021087999145189922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,2,128,1,fp8,fp8,0,0.021295999487241108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,4,128,1,float16,fp8,0,0.01858666663368543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,4,128,1,float16,float16,0,0.017957333475351334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,4,128,1,fp8,fp8,0,0.021589333812395733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,8,128,1,float16,float16,0,0.01802666609485944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,8,128,1,fp8,fp8,0,0.02197866638501485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,32,8,128,1,float16,fp8,0,0.018378666291634243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,32,128,1,float16,fp8,0,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,32,128,1,fp8,fp8,0,0.017866666118303936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,1,128,1,float16,fp8,0,0.015397333850463232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,1,128,1,float16,float16,0,0.01469333345691363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,32,128,1,float16,float16,0,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,1,128,1,fp8,fp8,0,0.01729600007335345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,2,128,1,float16,float16,0,0.014725333700577417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,2,128,1,float16,fp8,0,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,4,128,1,float16,fp8,0,0.015450666348139444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,4,128,1,float16,float16,0,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,4,128,1,fp8,fp8,0,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,2,128,1,fp8,fp8,0,0.017231999586025875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,8,128,1,float16,fp8,0,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,8,128,1,fp8,fp8,0,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,32,128,1,float16,fp8,0,0.014581333845853806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,32,8,128,1,float16,float16,0,0.015114666273196539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,32,128,1,fp8,fp8,0,0.016783999900023144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,32,128,1,float16,float16,0,0.014256000518798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,1,128,1,float16,float16,0,0.01421333352724711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,1,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,2,128,1,float16,float16,0,0.014432000617186228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,1,128,1,fp8,fp8,0,0.016714667280515034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,2,128,1,float16,fp8,0,0.014469332993030548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,4,128,1,float16,float16,0,0.01434133326013883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,4,128,1,fp8,fp8,0,0.01672533278663953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,4,128,1,float16,fp8,0,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,8,128,1,float16,float16,0,0.014362666755914688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,8,128,1,float16,fp8,0,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,2,128,1,fp8,fp8,0,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,32,8,128,1,fp8,fp8,0,0.016783999900023144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,32,128,1,float16,float16,0,0.01421333352724711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,32,128,1,float16,fp8,0,0.014741333822409311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,32,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,1,128,1,float16,fp8,0,0.014826666563749313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,1,128,1,fp8,fp8,0,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,1,128,1,float16,float16,0,0.014096000542243322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,2,128,1,fp8,fp8,0,0.016165333489576977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,2,128,1,float16,fp8,0,0.014815999815861383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,4,128,1,float16,float16,0,0.01423466702302297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,2,128,1,float16,float16,0,0.014416000495354334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,4,128,1,float16,fp8,0,0.01482133318980535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,4,128,1,fp8,fp8,0,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,8,128,1,float16,float16,0,0.014175999909639359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,8,128,1,fp8,fp8,0,0.016303999970356624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,32,8,128,1,float16,fp8,0,0.01444799949725469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,1,128,1,float16,fp8,0,24.787353515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,1,128,1,fp8,fp8,0,15.920655568440756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,1,128,1,float16,float16,0,24.6857172648112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,2,128,1,fp8,fp8,0,15.809429168701172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,2,128,1,float16,fp8,0,24.218592325846355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,2,128,1,float16,float16,0,24.60316212972005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,4,128,1,float16,float16,0,24.573130289713543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,4,128,1,float16,fp8,0,25.174021402994793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,1,128,1,float16,float16,0,12.649818420410156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,4,128,1,fp8,fp8,0,16.073221842447918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,1,128,1,float16,fp8,0,12.566757202148438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,24,128,1,float16,float16,0,11.976378122965494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,24,128,1,float16,fp8,0,12.029120127360025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,8,128,1,float16,fp8,0,24.663551330566406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,24,128,1,fp8,fp8,0,7.969674428304036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,8,128,1,float16,float16,0,24.8867670694987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,24,8,128,1,fp8,fp8,0,15.976144154866537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,1,128,1,fp8,fp8,0,8.124554951985678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,2,128,1,float16,float16,0,12.69879404703776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,2,128,1,float16,fp8,0,12.205499013264975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,2,128,1,fp8,fp8,0,7.901119867960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,4,128,1,float16,float16,0,12.4126828511556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,4,128,1,float16,fp8,0,12.283765157063803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,4,128,1,fp8,fp8,0,7.956549326578776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,8,128,1,float16,float16,0,12.493935902913412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,1,128,1,float16,float16,0,6.062794367472331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,8,128,1,float16,fp8,0,12.607835133870443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,1,128,1,float16,fp8,0,6.273637135823567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,1,128,1,fp8,fp8,0,3.974458694458008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,24,128,1,float16,float16,0,5.874373118082683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,24,8,128,1,fp8,fp8,0,8.230703989664713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,24,128,1,float16,fp8,0,6.044645309448242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,24,128,1,fp8,fp8,0,3.8676586151123047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,2,128,1,float16,fp8,0,5.852389017740886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,2,128,1,float16,float16,0,6.190101623535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,2,128,1,fp8,fp8,0,3.9104639689127603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,4,128,1,float16,float16,0,6.201973597208659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,4,128,1,float16,fp8,0,6.163103739420573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,4,128,1,fp8,fp8,0,4.032997449239095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,24,128,1,float16,float16,0,2.9913813273111978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,8,128,1,float16,fp8,0,6.013674418131511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,8,128,1,fp8,fp8,0,3.9867518742879233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,1,128,1,float16,float16,0,3.074320157368978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,24,128,1,float16,fp8,0,2.9791574478149414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,24,8,128,1,float16,float16,0,6.059802373250325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,1,128,1,float16,fp8,0,3.0904159545898438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,1,128,1,fp8,fp8,0,2.089183966318766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,24,128,1,fp8,fp8,0,2.0490293502807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,2,128,1,float16,fp8,0,3.0480639139811196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,2,128,1,fp8,fp8,0,2.0904693603515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,2,128,1,float16,float16,0,3.0749279657999673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,4,128,1,float16,fp8,0,3.103679974873861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,4,128,1,float16,float16,0,3.08024533589681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,4,128,1,fp8,fp8,0,2.101439952850342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,8,128,1,float16,fp8,0,3.057248115539551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,8,128,1,float16,float16,0,3.072085380554199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,24,8,128,1,fp8,fp8,0,2.117578665415446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,1,128,1,float16,fp8,0,14.312272389729818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,1,128,1,float16,float16,0,14.076095581054688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,1,128,1,fp8,fp8,0,9.301738739013672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,2,128,1,float16,float16,0,14.321637471516928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,2,128,1,float16,fp8,0,14.451802571614584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,2,128,1,fp8,fp8,0,9.368207931518555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,4,128,1,float16,float16,0,14.185653686523438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,4,128,1,float16,fp8,0,14.354122161865234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,1,128,1,float16,float16,0,7.268063863118489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,4,128,1,fp8,fp8,0,9.42630386352539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,8,128,1,float16,float16,0,14.230032602945963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,8,128,1,float16,fp8,0,14.306709289550781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,24,128,1,float16,float16,0,7.071861267089844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,1,128,1,float16,fp8,0,7.089093526204427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,1,128,1,fp8,fp8,0,4.526341438293457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,24,128,1,float16,fp8,0,6.9537811279296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,24,128,1,fp8,fp8,0,4.489397366841634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,2,128,1,float16,float16,0,6.9560801188151045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,24,8,128,1,fp8,fp8,0,9.310133616129557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,2,128,1,float16,fp8,0,6.996874491373698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,4,128,1,float16,fp8,0,7.0418135325113935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,4,128,1,fp8,fp8,0,4.674938519795735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,4,128,1,float16,float16,0,7.1583677927653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,2,128,1,fp8,fp8,0,4.5423892339070635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,1,128,1,float16,float16,0,3.462928136189779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,1,128,1,float16,fp8,0,3.5452534357706704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,8,128,1,float16,float16,0,7.128469467163086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,8,128,1,float16,fp8,0,7.1584428151448565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,24,128,1,float16,float16,0,3.4916321436564126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,1,128,1,fp8,fp8,0,2.3653066953023276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,24,8,128,1,fp8,fp8,0,4.670538584391276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,24,128,1,float16,fp8,0,3.480047861735026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,24,128,1,fp8,fp8,0,2.3347840309143066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,2,128,1,float16,float16,0,3.5648587544759116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,2,128,1,float16,fp8,0,3.403717358907064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,2,128,1,fp8,fp8,0,2.3463892936706543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,4,128,1,float16,float16,0,3.4607359568277993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,4,128,1,float16,fp8,0,3.4550399780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,4,128,1,fp8,fp8,0,2.3477867444356284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,24,128,1,float16,float16,0,1.7980693181355794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,8,128,1,fp8,fp8,0,2.379525343577067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,8,128,1,float16,float16,0,3.535050710042318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,24,8,128,1,float16,fp8,0,3.4955625534057617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,24,128,1,float16,fp8,0,1.779568036397298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,1,128,1,float16,float16,0,1.841418743133545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,1,128,1,float16,fp8,0,1.8121120134989421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,1,128,1,fp8,fp8,0,1.2741653124491374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,2,128,1,float16,float16,0,1.8617013295491536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,24,128,1,fp8,fp8,0,1.26528000831604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,2,128,1,float16,fp8,0,1.822879950205485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,2,128,1,fp8,fp8,0,1.2762347062428792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,4,128,1,float16,float16,0,1.8357547124226887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,4,128,1,float16,fp8,0,1.8277494112650554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,4,128,1,fp8,fp8,0,1.2783413728078206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,8,128,1,float16,float16,0,1.8401813507080078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,8,128,1,fp8,fp8,0,1.2866079807281494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,24,8,128,1,float16,fp8,0,1.8546026547749836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,1,128,1,fp8,fp8,0,6.611946741739909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,1,128,1,float16,fp8,0,10.112016042073568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,1,128,1,float16,float16,0,10.1845334370931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,2,128,1,float16,float16,0,10.291664123535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,2,128,1,fp8,fp8,0,6.621146519978841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,2,128,1,float16,fp8,0,10.063599904378256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,4,128,1,float16,fp8,0,10.262863794962565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,4,128,1,float16,float16,0,10.165840148925781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,1,128,1,float16,float16,0,4.968458811442058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,4,128,1,fp8,fp8,0,6.641477584838867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,8,128,1,float16,fp8,0,10.141295750935873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,24,128,1,float16,float16,0,5.019280115763347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,8,128,1,float16,float16,0,10.26476796468099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,24,8,128,1,fp8,fp8,0,6.755034764607747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,1,128,1,fp8,fp8,0,3.256160100301107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,24,128,1,float16,fp8,0,4.934362729390462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,24,128,1,fp8,fp8,0,3.3346665700276694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,1,128,1,float16,fp8,0,5.003525416056315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,2,128,1,float16,fp8,0,4.930202802022298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,2,128,1,float16,float16,0,4.895872116088867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,2,128,1,fp8,fp8,0,3.2560907999674478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,4,128,1,float16,float16,0,5.124234517415364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,4,128,1,float16,fp8,0,4.910250663757324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,4,128,1,fp8,fp8,0,3.2702614466349282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,8,128,1,float16,float16,0,4.939514795939128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,8,128,1,float16,fp8,0,4.883664131164551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,1,128,1,float16,float16,0,2.4663146336873374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,24,128,1,float16,float16,0,2.4578986167907715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,24,128,1,float16,fp8,0,2.4327573776245117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,24,8,128,1,fp8,fp8,0,3.3187840779622397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,24,128,1,fp8,fp8,0,1.754634698232015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,1,128,1,float16,fp8,0,2.4685866038004556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,1,128,1,fp8,fp8,0,1.7106506029764812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,2,128,1,float16,float16,0,2.4839040438334146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,2,128,1,float16,fp8,0,2.444437344868978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,2,128,1,fp8,fp8,0,1.710319995880127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,4,128,1,float16,float16,0,2.4812960624694824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,4,128,1,float16,fp8,0,2.4514506657918296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,4,128,1,fp8,fp8,0,1.7339146931966145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,8,128,1,float16,fp8,0,2.514906724294027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,8,128,1,float16,float16,0,2.4873387018839517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,1,128,1,float16,float16,0,1.3314879735310872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,1,128,1,float16,fp8,0,1.3225119908650715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,24,8,128,1,fp8,fp8,0,1.7280319531758626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,24,128,1,float16,fp8,0,1.3072960376739502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,24,128,1,float16,float16,0,1.3084479967753093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,24,128,1,fp8,fp8,0,0.9379200140635172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,1,128,1,fp8,fp8,0,0.94159468015035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,2,128,1,float16,fp8,0,1.3190986315409343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,2,128,1,fp8,fp8,0,0.9490133126576742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,2,128,1,float16,float16,0,1.334112008412679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,4,128,1,fp8,fp8,0,0.9468159675598145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,4,128,1,float16,fp8,0,1.3247839609781902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,4,128,1,float16,float16,0,1.3402187029520671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,8,128,1,float16,float16,0,1.3419520060221355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,8,128,1,fp8,fp8,0,0.9504426320393881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,24,8,128,1,float16,fp8,0,1.3450667063395183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,1,128,1,float16,float16,0,13.578234354654947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,1,128,1,fp8,fp8,0,8.78766949971517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,1,128,1,float16,fp8,0,13.27676773071289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,2,128,1,float16,float16,0,13.373221079508463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,2,128,1,fp8,fp8,0,8.748048146565756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,4,128,1,float16,float16,0,13.30614980061849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,4,128,1,float16,fp8,0,13.579573313395182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,2,128,1,float16,fp8,0,13.349183400472006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,4,128,1,fp8,fp8,0,9.05734380086263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,1,128,1,float16,float16,0,6.621909459431966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,8,128,1,float16,float16,0,13.291839599609375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,24,128,1,float16,float16,0,6.651866912841797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,8,128,1,fp8,fp8,0,8.965290705362955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,1,128,1,float16,fp8,0,6.570112228393555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,24,8,128,1,float16,fp8,0,13.287775675455729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,24,128,1,float16,fp8,0,6.422133127848308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,1,128,1,fp8,fp8,0,4.3807573318481445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,24,128,1,fp8,fp8,0,4.557861328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,2,128,1,float16,float16,0,6.316938400268555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,2,128,1,fp8,fp8,0,4.402304013570149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,2,128,1,float16,fp8,0,6.593626658121745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,4,128,1,float16,float16,0,6.924474716186523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,4,128,1,fp8,fp8,0,4.319456100463867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,4,128,1,float16,fp8,0,6.67799441019694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,8,128,1,float16,float16,0,6.652074813842773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,1,128,1,float16,float16,0,3.174799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,8,128,1,float16,fp8,0,6.7223256429036455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,1,128,1,float16,fp8,0,3.1403414408365884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,24,8,128,1,fp8,fp8,0,4.436730702718099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,24,128,1,float16,float16,0,3.2007360458374023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,24,128,1,float16,fp8,0,3.171680132548014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,1,128,1,fp8,fp8,0,2.202378590901693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,24,128,1,fp8,fp8,0,2.339637279510498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,2,128,1,float16,float16,0,3.1788107554117837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,2,128,1,float16,fp8,0,3.227317492167155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,2,128,1,fp8,fp8,0,2.204730669657389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,4,128,1,float16,float16,0,3.18611749013265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,4,128,1,fp8,fp8,0,2.220469315846761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,4,128,1,float16,fp8,0,3.167477289835612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,8,128,1,float16,float16,0,3.300325393676758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,24,128,1,float16,float16,0,1.683695952097575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,8,128,1,fp8,fp8,0,2.24342934290568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,1,128,1,float16,float16,0,1.6647680600484211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,24,8,128,1,float16,fp8,0,3.2480481465657554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,1,128,1,float16,fp8,0,1.646714687347412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,24,128,1,float16,fp8,0,1.6409866015116374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,1,128,1,fp8,fp8,0,1.1688586870829265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,2,128,1,float16,float16,0,1.689733346303304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,24,128,1,fp8,fp8,0,1.1861653327941895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,2,128,1,fp8,fp8,0,1.170255978902181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,2,128,1,float16,fp8,0,1.645055929819743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,4,128,1,fp8,fp8,0,1.1753919919331868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,4,128,1,float16,float16,0,1.6968746185302734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,4,128,1,float16,fp8,0,1.654853343963623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,1,128,1,float16,fp8,0,0.9018293221791586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,1,128,1,float16,float16,0,0.9198079903920492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,8,128,1,float16,float16,0,1.684597333272298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,24,128,1,float16,float16,0,0.902890682220459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,8,128,1,fp8,fp8,0,1.1902506351470947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,24,8,128,1,float16,fp8,0,1.6852854092915852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,24,128,1,float16,fp8,0,0.8922560214996338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,1,128,1,fp8,fp8,0,0.6223253409067789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,2,128,1,float16,float16,0,0.9241706530253092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,24,128,1,fp8,fp8,0,0.6364373366038004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,2,128,1,float16,fp8,0,0.9040959676106771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,2,128,1,fp8,fp8,0,0.6234826644261678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,4,128,1,float16,fp8,0,0.9144906997680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,4,128,1,float16,float16,0,0.9172213077545166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,4,128,1,fp8,fp8,0,0.6253066857655843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,8,128,1,float16,float16,0,0.9213439623514811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,8,128,1,float16,fp8,0,0.9163146813710531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,24,8,128,1,fp8,fp8,0,0.634773333867391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,1,128,1,fp8,fp8,0,5.247829437255859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,1,128,1,float16,float16,0,7.859861373901367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,1,128,1,float16,fp8,0,7.5972639719645185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,2,128,1,float16,float16,0,7.723477045694987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,2,128,1,float16,fp8,0,7.691237131754558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,2,128,1,fp8,fp8,0,5.26911989847819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,4,128,1,float16,float16,0,8.16934903462728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,4,128,1,float16,fp8,0,7.631231943766276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,1,128,1,float16,float16,0,3.821903864542643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,4,128,1,fp8,fp8,0,5.361930847167969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,8,128,1,float16,float16,0,8.0535519917806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,8,128,1,float16,fp8,0,7.904650370279948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,24,8,128,1,fp8,fp8,0,5.268575986226399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,24,128,1,float16,float16,0,3.8998826344807944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,24,128,1,float16,fp8,0,3.788127899169922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,24,128,1,fp8,fp8,0,2.8896748224894204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,1,128,1,float16,fp8,0,3.7679147720336914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,1,128,1,fp8,fp8,0,2.6319519678751626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,2,128,1,float16,float16,0,3.753727912902832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,2,128,1,fp8,fp8,0,2.632474740346273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,2,128,1,float16,fp8,0,3.7793706258138022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,4,128,1,float16,fp8,0,3.819706598917643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,4,128,1,float16,float16,0,3.837050755818685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,4,128,1,fp8,fp8,0,2.744175910949707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,8,128,1,float16,float16,0,3.8453121185302734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,8,128,1,float16,fp8,0,3.8257439931233725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,1,128,1,float16,float16,0,1.9104426701863606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,1,128,1,float16,fp8,0,1.9128959973653157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,1,128,1,fp8,fp8,0,1.3563040097554524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,2,128,1,float16,float16,0,1.9446400006612141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,24,128,1,float16,float16,0,1.925322691599528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,24,8,128,1,fp8,fp8,0,2.68391482035319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,24,128,1,fp8,fp8,0,1.409114678700765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,24,128,1,float16,fp8,0,1.917893409729004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,2,128,1,float16,fp8,0,1.8885919253031414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,2,128,1,fp8,fp8,0,1.3869919776916504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,4,128,1,float16,float16,0,1.9217599232991536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,4,128,1,float16,fp8,0,1.9282933870951335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,4,128,1,fp8,fp8,0,1.3719786008199055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,24,128,1,float16,float16,0,1.01801602045695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,8,128,1,float16,float16,0,1.9433600107828777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,24,128,1,float16,fp8,0,1.0597386360168457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,8,128,1,fp8,fp8,0,1.3870666821797688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,24,8,128,1,float16,fp8,0,1.9158986409505208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,1,128,1,float16,float16,0,1.0281600157419841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,1,128,1,float16,fp8,0,1.0022400220235188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,1,128,1,fp8,fp8,0,0.7321706612904867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,24,128,1,fp8,fp8,0,0.7521653175354004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,2,128,1,float16,float16,0,1.0216906865437825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,2,128,1,float16,fp8,0,1.017584005991618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,2,128,1,fp8,fp8,0,0.7335893313090006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,4,128,1,float16,float16,0,1.023530642191569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,4,128,1,float16,fp8,0,1.020624001820882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,4,128,1,fp8,fp8,0,0.739578644434611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,1,128,1,float16,float16,0,0.5754986604054769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,8,128,1,float16,float16,0,1.0263946851094563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,1,128,1,float16,fp8,0,0.5650240182876587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,24,128,1,float16,float16,0,0.5848906834920248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,8,128,1,fp8,fp8,0,0.7481066385904948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,24,128,1,float16,fp8,0,0.561626672744751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,24,128,1,fp8,fp8,0,0.4099573294321696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,24,8,128,1,float16,fp8,0,1.012288014094035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,1,128,1,fp8,fp8,0,0.3954933484395345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,2,128,1,float16,fp8,0,0.5658239920934042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,2,128,1,float16,float16,0,0.5750879844029745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,2,128,1,fp8,fp8,0,0.40252800782521564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,4,128,1,float16,float16,0,0.5767199993133545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,4,128,1,float16,fp8,0,0.5669013261795044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,4,128,1,fp8,fp8,0,0.3981066544850667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,8,128,1,float16,float16,0,0.5770346721013387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,8,128,1,float16,fp8,0,0.5723413228988647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,24,8,128,1,fp8,fp8,0,0.40425066153208417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,1,128,1,float16,float16,0,7.67195192972819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,1,128,1,fp8,fp8,0,5.23525873819987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,1,128,1,float16,fp8,0,7.919567743937175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,2,128,1,fp8,fp8,0,5.256138801574707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,2,128,1,float16,fp8,0,7.515829086303711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,2,128,1,float16,float16,0,7.7163041432698565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,4,128,1,float16,fp8,0,7.528170903523763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,4,128,1,float16,float16,0,7.842714945475261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,1,128,1,float16,float16,0,3.77401065826416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,4,128,1,fp8,fp8,0,5.308570543924968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,8,128,1,float16,float16,0,7.7870025634765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,1,128,1,float16,fp8,0,3.7118399937947593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,8,128,1,float16,fp8,0,8.042954762776693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,24,128,1,fp8,fp8,0,2.798938751220703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,24,128,1,float16,float16,0,3.807765324910482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,24,128,1,float16,fp8,0,3.7863359451293945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,1,128,1,fp8,fp8,0,2.6470185915629068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,24,8,128,1,fp8,fp8,0,5.394554773966472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,2,128,1,float16,float16,0,3.69484806060791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,2,128,1,float16,fp8,0,3.660005251566569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,2,128,1,fp8,fp8,0,2.8175840377807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,4,128,1,float16,fp8,0,3.739765485127767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,4,128,1,float16,float16,0,3.6841599146525064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,4,128,1,fp8,fp8,0,2.6833012898763022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,8,128,1,float16,float16,0,3.791685422261556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,1,128,1,float16,float16,0,1.849679946899414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,8,128,1,float16,fp8,0,3.663386662801107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,1,128,1,float16,fp8,0,1.8157706260681152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,1,128,1,fp8,fp8,0,1.3490613301595051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,24,128,1,float16,float16,0,1.953839937845866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,2,128,1,float16,float16,0,1.855829397837321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,24,128,1,float16,fp8,0,2.071839968363444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,2,128,1,fp8,fp8,0,1.3525546391805012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,2,128,1,float16,fp8,0,1.819493293762207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,24,128,1,fp8,fp8,0,1.4365065892537434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,24,8,128,1,fp8,fp8,0,2.72977606455485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,4,128,1,float16,float16,0,1.8618987401326497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,4,128,1,float16,fp8,0,1.8573333422342937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,4,128,1,fp8,fp8,0,1.3723947207132976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,24,128,1,float16,float16,0,0.9958453178405762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,8,128,1,float16,float16,0,1.9767039616902669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,8,128,1,float16,fp8,0,1.85098663965861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,24,8,128,1,fp8,fp8,0,1.3924800554911296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,24,128,1,float16,fp8,0,0.9820640087127686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,1,128,1,float16,float16,0,0.9741600354512533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,1,128,1,float16,fp8,0,0.9507733186086019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,1,128,1,fp8,fp8,0,0.7134772936503092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,24,128,1,fp8,fp8,0,0.7577226956685384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,2,128,1,float16,float16,0,0.9947893619537354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,2,128,1,float16,fp8,0,0.9523466428120931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,2,128,1,fp8,fp8,0,0.7145600318908691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,4,128,1,fp8,fp8,0,0.7224586804707845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,4,128,1,float16,float16,0,0.9749866326649984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,4,128,1,float16,fp8,0,0.9666986465454102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,8,128,1,float16,float16,0,0.9827466805775961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,8,128,1,float16,fp8,0,0.9667733510335287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,24,128,1,float16,float16,0,0.5333439906438192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,24,8,128,1,fp8,fp8,0,0.759008010228475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,1,128,1,float16,fp8,0,0.5194133520126343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,24,128,1,float16,fp8,0,0.5250933170318604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,1,128,1,float16,float16,0,0.529423991839091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,24,128,1,fp8,fp8,0,0.41230932871500653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,1,128,1,fp8,fp8,0,0.37861335277557373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,2,128,1,float16,float16,0,0.5307199954986572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,2,128,1,float16,fp8,0,0.5196640094121298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,2,128,1,fp8,fp8,0,0.3887573480606079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,4,128,1,float16,fp8,0,0.5216960112253824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,4,128,1,float16,float16,0,0.5330880085627238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,8,128,1,float16,float16,0,0.5358133316040039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,4,128,1,fp8,fp8,0,0.3818826675415039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,8,128,1,float16,fp8,0,0.527786652247111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,24,8,128,1,fp8,fp8,0,0.3911093473434448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,24,128,1,float16,float16,0,0.28538666168848675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,24,128,1,float16,fp8,0,0.291098674138387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,1,128,1,float16,fp8,0,0.2722826600074768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,2,128,1,float16,float16,0,0.2797386646270752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,2,128,1,float16,fp8,0,0.2725653251012166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,1,128,1,float16,float16,0,0.27780266602834064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,2,128,1,fp8,fp8,0,0.21912533044815063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,4,128,1,float16,float16,0,0.28324800729751587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,24,128,1,fp8,fp8,0,0.22940266132354736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,1,128,1,fp8,fp8,0,0.21888534228006998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,4,128,1,float16,fp8,0,0.2737226684888204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,8,128,1,float16,float16,0,0.28173333406448364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,4,128,1,fp8,fp8,0,0.2220319906870524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,8,128,1,fp8,fp8,0,0.223306675752004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,24,8,128,1,float16,fp8,0,0.27560534079869586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,1,128,1,float16,float16,0,4.89027722676595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,1,128,1,fp8,fp8,0,3.3381118774414062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,1,128,1,float16,fp8,0,4.492095947265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,2,128,1,float16,float16,0,4.529786745707194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,2,128,1,fp8,fp8,0,3.361407915751139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,2,128,1,float16,fp8,0,4.544837315877278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,4,128,1,float16,float16,0,4.68174393971761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,4,128,1,float16,fp8,0,4.60912545522054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,1,128,1,float16,float16,0,2.2850186030069985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,4,128,1,fp8,fp8,0,3.6623414357503257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,24,128,1,float16,float16,0,2.3957974116007485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,8,128,1,float16,float16,0,4.64357852935791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,24,128,1,float16,fp8,0,2.3399786949157715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,8,128,1,float16,fp8,0,4.519248008728027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,24,8,128,1,fp8,fp8,0,3.4538399378458657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,1,128,1,float16,fp8,0,2.2358667055765786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,1,128,1,fp8,fp8,0,1.7800639470418294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,24,128,1,fp8,fp8,0,1.8201972643534343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,2,128,1,float16,float16,0,2.268085320790609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,2,128,1,float16,fp8,0,2.220341364542643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,2,128,1,fp8,fp8,0,1.6958932876586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,4,128,1,float16,fp8,0,2.2298453648885093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,4,128,1,float16,float16,0,2.2792372703552246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,4,128,1,fp8,fp8,0,1.7141493161519368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,8,128,1,float16,float16,0,2.511082649230957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,1,128,1,float16,float16,0,1.1537813345591228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,8,128,1,float16,fp8,0,2.2620372772216797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,1,128,1,float16,fp8,0,1.1401387055714924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,24,128,1,float16,fp8,0,1.1929439703623455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,24,8,128,1,fp8,fp8,0,1.749077320098877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,1,128,1,fp8,fp8,0,0.8938612937927246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,24,128,1,float16,float16,0,1.222218672434489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,24,128,1,fp8,fp8,0,0.941642681757609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,2,128,1,float16,float16,0,1.1582400004069011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,2,128,1,float16,fp8,0,1.1361707051595051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,2,128,1,fp8,fp8,0,0.8701173464457194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,4,128,1,float16,float16,0,1.1656053066253662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,4,128,1,float16,fp8,0,1.1430026690165203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,4,128,1,fp8,fp8,0,0.9103679656982422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,8,128,1,float16,fp8,0,1.1559200286865234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,8,128,1,float16,float16,0,1.1804052988688152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,1,128,1,float16,float16,0,0.6127306620279948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,1,128,1,float16,fp8,0,0.6141759951909384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,24,128,1,float16,float16,0,0.6346293290456136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,24,8,128,1,fp8,fp8,0,0.8968266646067301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,24,128,1,float16,fp8,0,0.6302239894866943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,1,128,1,fp8,fp8,0,0.4678293466567993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,24,128,1,fp8,fp8,0,0.504319985707601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,2,128,1,fp8,fp8,0,0.48076800505320233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,2,128,1,float16,fp8,0,0.6022026538848877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,2,128,1,float16,float16,0,0.6142986615498861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,4,128,1,float16,float16,0,0.6174293359120687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,4,128,1,float16,fp8,0,0.6061919927597046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,4,128,1,fp8,fp8,0,0.4717173178990682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,8,128,1,float16,float16,0,0.6252586841583252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,8,128,1,float16,fp8,0,0.6117120186487833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,24,8,128,1,fp8,fp8,0,0.49861868222554523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,24,128,1,float16,float16,0,0.3498133420944214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,24,128,1,fp8,fp8,0,0.2748426596323649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,1,128,1,fp8,fp8,0,0.2504640022913615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,1,128,1,float16,float16,0,0.34221335252126056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,2,128,1,float16,float16,0,0.343669335047404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,24,128,1,float16,fp8,0,0.34508800506591797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,1,128,1,float16,fp8,0,0.3346773386001587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,2,128,1,float16,fp8,0,0.3434613148371379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,2,128,1,fp8,fp8,0,0.25170133511225384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,4,128,1,float16,float16,0,0.3451840082804362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,4,128,1,float16,fp8,0,0.338645339012146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,4,128,1,fp8,fp8,0,0.25387199719746906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,8,128,1,float16,float16,0,0.34784531593322754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,24,128,1,float16,float16,0,0.2016693353652954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,8,128,1,fp8,fp8,0,0.2584266662597656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,24,128,1,float16,fp8,0,0.18712000052134195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,1,128,1,float16,float16,0,0.18413333098093668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,24,8,128,1,float16,fp8,0,0.3421440124511719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,24,128,1,fp8,fp8,0,0.15825600425402322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,1,128,1,float16,fp8,0,0.17972266674041748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,1,128,1,fp8,fp8,0,0.1478613317012787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,2,128,1,float16,float16,0,0.18654932578404745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,2,128,1,float16,fp8,0,0.18023999532063803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,2,128,1,fp8,fp8,0,0.14869866768519083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,4,128,1,float16,float16,0,0.18587199846903482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,4,128,1,fp8,fp8,0,0.15005866686503092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,4,128,1,float16,fp8,0,0.18133866786956787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,8,128,1,fp8,fp8,0,0.15683733423550925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,8,128,1,float16,fp8,0,0.18287465969721475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,24,8,128,1,float16,float16,0,0.1872373421986898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,1,128,1,fp8,fp8,0,3.703237215677897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,1,128,1,float16,float16,0,4.861184120178223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,1,128,1,float16,fp8,0,4.7147626876831055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,2,128,1,float16,fp8,0,4.638378779093425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,2,128,1,fp8,fp8,0,4.097797393798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,4,128,1,float16,float16,0,4.950959841410319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,2,128,1,float16,float16,0,4.828997294108073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,4,128,1,float16,fp8,0,4.694213231404622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,1,128,1,float16,float16,0,2.3831146558125815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,4,128,1,fp8,fp8,0,3.770469347635905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,8,128,1,float16,float16,0,5.023557345072429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,24,128,1,float16,float16,0,3.0038185119628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,24,128,1,float16,fp8,0,2.4896480242411294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,1,128,1,float16,fp8,0,2.29422934850057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,8,128,1,float16,fp8,0,4.753498713175456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,24,128,1,fp8,fp8,0,2.039541403452555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,24,8,128,1,fp8,fp8,0,3.8285226821899414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,1,128,1,fp8,fp8,0,1.8466827074686687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,2,128,1,float16,float16,0,2.4070986111958823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,2,128,1,float16,fp8,0,2.5246240297953286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,2,128,1,fp8,fp8,0,1.8614826202392578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,4,128,1,float16,float16,0,2.4158719380696616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,4,128,1,float16,fp8,0,2.3274505933125815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,4,128,1,fp8,fp8,0,1.8790987332661946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,8,128,1,float16,fp8,0,2.3635360399881997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,8,128,1,float16,float16,0,2.455440044403076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,1,128,1,float16,fp8,0,1.1560906569163005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,1,128,1,float16,float16,0,1.190293312072754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,24,128,1,float16,float16,0,1.5125172932942708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,24,8,128,1,fp8,fp8,0,1.9183573722839355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,1,128,1,fp8,fp8,0,0.932096004486084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,24,128,1,float16,fp8,0,1.2613759835561116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,2,128,1,float16,fp8,0,1.1592480341593425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,2,128,1,float16,float16,0,1.1935893694559734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,24,128,1,fp8,fp8,0,1.0385653177897136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,2,128,1,fp8,fp8,0,0.9385920365651449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,4,128,1,float16,float16,0,1.280512015024821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,4,128,1,fp8,fp8,0,0.9519306818644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,8,128,1,float16,float16,0,1.2219200134277344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,4,128,1,float16,fp8,0,1.1790986855824788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,1,128,1,float16,float16,0,0.6217813491821289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,8,128,1,float16,fp8,0,1.18886399269104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,24,128,1,float16,float16,0,0.6642080148061117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,1,128,1,float16,fp8,0,0.604202667872111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,24,128,1,float16,fp8,0,0.6519360144933065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,1,128,1,fp8,fp8,0,0.4888906478881836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,24,8,128,1,fp8,fp8,0,0.97270401318868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,24,128,1,fp8,fp8,0,0.6086560090382894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,2,128,1,float16,float16,0,0.6232159932454427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,4,128,1,float16,float16,0,0.6267999807993571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,2,128,1,float16,fp8,0,0.6072426637013754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,2,128,1,fp8,fp8,0,0.49060265223185223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,4,128,1,float16,fp8,0,0.6132959922154745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,4,128,1,fp8,fp8,0,0.49747200806935626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,8,128,1,float16,float16,0,0.6351253191630045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,8,128,1,float16,fp8,0,0.658245325088501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,24,8,128,1,fp8,fp8,0,0.5101279815038046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,1,128,1,float16,float16,0,0.3359626531600952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,24,128,1,float16,float16,0,0.35450132687886554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,24,128,1,float16,fp8,0,0.3475253184636434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,1,128,1,float16,fp8,0,0.32740267117818195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,2,128,1,float16,float16,0,0.3506400187810262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,1,128,1,fp8,fp8,0,0.2589813272158305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,2,128,1,float16,fp8,0,0.3299413323402405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,2,128,1,fp8,fp8,0,0.26155199607213336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,24,128,1,fp8,fp8,0,0.30208534002304077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,4,128,1,float16,float16,0,0.3399306535720825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,4,128,1,float16,fp8,0,0.3310133417447408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,8,128,1,float16,float16,0,0.36611199378967285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,24,128,1,float16,float16,0,0.19594667355219522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,4,128,1,fp8,fp8,0,0.2629759907722473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,8,128,1,fp8,fp8,0,0.27350399891535443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,24,8,128,1,float16,fp8,0,0.33608531951904297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,24,128,1,float16,fp8,0,0.1909173329671224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,24,128,1,fp8,fp8,0,0.16237866878509521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,1,128,1,float16,float16,0,0.1797706683476766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,1,128,1,float16,fp8,0,0.1779200037320455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,1,128,1,fp8,fp8,0,0.14778133233388266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,2,128,1,float16,fp8,0,0.1761066714922587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,2,128,1,float16,float16,0,0.17991999785105386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,2,128,1,fp8,fp8,0,0.14686399698257446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,4,128,1,float16,float16,0,0.18125333388646445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,4,128,1,float16,fp8,0,0.1770240068435669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,4,128,1,fp8,fp8,0,0.14893866578737894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,8,128,1,fp8,fp8,0,0.15290133158365884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,8,128,1,float16,float16,0,0.19252800941467285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,24,128,1,float16,float16,0,0.11180800199508667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,1,128,1,float16,fp8,0,0.10482133428255717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,24,8,128,1,float16,fp8,0,0.18069867293039957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,1,128,1,float16,float16,0,0.10641599694887798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,24,128,1,float16,fp8,0,0.1095306674639384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,24,128,1,fp8,fp8,0,0.09867733716964722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,1,128,1,fp8,fp8,0,0.08746666709582011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,2,128,1,float16,float16,0,0.10644800464312236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,2,128,1,float16,fp8,0,0.10523733496665955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,2,128,1,fp8,fp8,0,0.08839999636014302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,4,128,1,float16,fp8,0,0.10594666997591655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,4,128,1,float16,float16,0,0.10820266604423523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,4,128,1,fp8,fp8,0,0.09073066711425781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,8,128,1,float16,fp8,0,0.1074026624361674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,8,128,1,fp8,fp8,0,0.09634666641553243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,24,8,128,1,float16,float16,0,0.10877866546312968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,1,128,1,float16,fp8,0,2.970597267150879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,1,128,1,fp8,fp8,0,2.4826879501342773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,1,128,1,float16,float16,0,3.11515744527181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,2,128,1,float16,float16,0,3.1060053507486978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,2,128,1,fp8,fp8,0,2.489664077758789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,2,128,1,float16,fp8,0,2.9854345321655273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,4,128,1,float16,fp8,0,3.0254294077555337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,4,128,1,float16,float16,0,3.5150718688964844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,1,128,1,float16,float16,0,1.662245273590088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,4,128,1,fp8,fp8,0,2.5361599922180176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,1,128,1,float16,fp8,0,1.6150186856587727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,24,128,1,float16,float16,0,1.699610710144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,8,128,1,float16,float16,0,3.214021364847819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,8,128,1,fp8,fp8,0,2.5964852968851724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,24,8,128,1,float16,fp8,0,3.091493288675944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,24,128,1,float16,fp8,0,1.6529706319173176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,24,128,1,fp8,fp8,0,1.4052160580952961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,1,128,1,fp8,fp8,0,1.240127960840861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,2,128,1,float16,float16,0,1.6811680793762207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,2,128,1,float16,fp8,0,1.488709290822347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,2,128,1,fp8,fp8,0,1.2550506591796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,4,128,1,float16,float16,0,1.5674452781677246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,4,128,1,fp8,fp8,0,1.269210656483968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,4,128,1,float16,fp8,0,1.5080053011576335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,8,128,1,float16,float16,0,1.592938741048177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,1,128,1,float16,fp8,0,0.7570186456044515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,1,128,1,float16,float16,0,0.7797280152638754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,8,128,1,float16,fp8,0,1.5367627143859863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,24,128,1,float16,float16,0,0.8589920202891032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,24,128,1,float16,fp8,0,0.8374613126118978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,24,8,128,1,fp8,fp8,0,1.4093759854634602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,2,128,1,float16,float16,0,0.7827839851379395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,1,128,1,fp8,fp8,0,0.6333973407745361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,24,128,1,fp8,fp8,0,0.7250026861826578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,2,128,1,float16,fp8,0,0.7591253121693929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,2,128,1,fp8,fp8,0,0.6391893227895101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,4,128,1,float16,float16,0,0.7908319632212321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,4,128,1,float16,fp8,0,0.7660373051961263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,4,128,1,fp8,fp8,0,0.6794613202412924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,8,128,1,float16,float16,0,0.803663969039917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,1,128,1,float16,float16,0,0.4107626676559448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,1,128,1,float16,fp8,0,0.4145066738128662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,24,128,1,float16,fp8,0,0.4373653332392375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,8,128,1,fp8,fp8,0,0.6647573312123617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,24,128,1,float16,float16,0,0.44839998086293537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,24,8,128,1,float16,fp8,0,0.7809706528981527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,24,128,1,fp8,fp8,0,0.3816373348236084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,1,128,1,fp8,fp8,0,0.3372906843821208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,2,128,1,float16,float16,0,0.41282665729522705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,2,128,1,float16,fp8,0,0.4188053210576375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,2,128,1,fp8,fp8,0,0.3396639823913574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,4,128,1,float16,float16,0,0.4174186786015828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,4,128,1,float16,fp8,0,0.4052373170852661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,8,128,1,float16,float16,0,0.4227199951807658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,4,128,1,fp8,fp8,0,0.3436160087585449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,8,128,1,float16,fp8,0,0.4134720166524251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,1,128,1,float16,float16,0,0.22565333048502603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,24,8,128,1,fp8,fp8,0,0.3524640003840129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,24,128,1,float16,float16,0,0.28461867570877075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,24,128,1,float16,fp8,0,0.23746132850646973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,1,128,1,float16,fp8,0,0.22022400299708048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,24,128,1,fp8,fp8,0,0.21015467246373495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,1,128,1,fp8,fp8,0,0.1795039971669515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,2,128,1,float16,float16,0,0.22789333264033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,2,128,1,float16,fp8,0,0.22130133708318075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,2,128,1,fp8,fp8,0,0.18023467063903809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,4,128,1,float16,float16,0,0.24080532789230347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,4,128,1,float16,fp8,0,0.22364266713460287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,4,128,1,fp8,fp8,0,0.18347734212875366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,1,128,1,float16,float16,0,0.1241919994354248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,1,128,1,float16,fp8,0,0.12155733505884807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,24,128,1,float16,float16,0,0.13703999916712442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,24,128,1,fp8,fp8,0,0.12746133406956991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,8,128,1,float16,float16,0,0.23316800594329834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,8,128,1,fp8,fp8,0,0.1885333259900411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,24,8,128,1,float16,fp8,0,0.22753600279490152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,24,128,1,float16,fp8,0,0.13290133078893027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,1,128,1,fp8,fp8,0,0.10419733325640361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,2,128,1,float16,float16,0,0.12444800138473511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,2,128,1,fp8,fp8,0,0.10890666643778484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,4,128,1,float16,float16,0,0.12609066565831503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,2,128,1,float16,fp8,0,0.1218986709912618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,4,128,1,float16,fp8,0,0.12258666753768921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,8,128,1,float16,float16,0,0.1272586683432261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,4,128,1,fp8,fp8,0,0.10731200377146403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,8,128,1,float16,fp8,0,0.12495999534924825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,24,128,1,float16,float16,0,0.08353599905967712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,24,128,1,float16,fp8,0,0.0795306662718455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,1,128,1,float16,float16,0,0.07688533266385396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,24,128,1,fp8,fp8,0,0.07379200061162312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,1,128,1,float16,fp8,0,0.07539199789365132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,24,8,128,1,fp8,fp8,0,0.11035199960072835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,2,128,1,float16,float16,0,0.0767626663049062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,2,128,1,float16,fp8,0,0.07506666580835979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,2,128,1,fp8,fp8,0,0.06499200065930684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,1,128,1,fp8,fp8,0,0.06491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,4,128,1,float16,fp8,0,0.07579199969768524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,8,128,1,float16,float16,0,0.0772213339805603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,4,128,1,float16,float16,0,0.07706666489442189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,4,128,1,fp8,fp8,0,0.06547733147939046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,8,128,1,float16,fp8,0,0.07643733421961467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,24,8,128,1,fp8,fp8,0,0.06832000116507213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,1,128,1,float16,float16,0,2.8736693064371743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,1,128,1,float16,fp8,0,2.864447911580404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,2,128,1,fp8,fp8,0,2.610922654469808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,1,128,1,fp8,fp8,0,2.45196262995402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,2,128,1,float16,float16,0,2.952224095662435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,2,128,1,float16,fp8,0,2.8799893061319985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,4,128,1,float16,float16,0,3.1267147064208984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,4,128,1,float16,fp8,0,3.256239891052246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,1,128,1,float16,float16,0,1.5680054028828938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,4,128,1,fp8,fp8,0,2.7902825673421225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,1,128,1,float16,fp8,0,1.5610826810201008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,24,128,1,float16,float16,0,1.857034683227539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,24,128,1,float16,fp8,0,1.7968106269836426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,8,128,1,float16,float16,0,3.0649280548095703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,8,128,1,float16,fp8,0,3.20143985748291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,24,128,1,fp8,fp8,0,1.5227039655049641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,1,128,1,fp8,fp8,0,1.2202880382537842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,24,8,128,1,fp8,fp8,0,2.818725268046061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,2,128,1,float16,float16,0,1.5891413688659668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,2,128,1,float16,fp8,0,1.4516639709472656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,4,128,1,float16,float16,0,1.5543732643127441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,4,128,1,float16,fp8,0,1.55512539545695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,2,128,1,fp8,fp8,0,1.2989493211110432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,4,128,1,fp8,fp8,0,1.405786673227946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,8,128,1,float16,float16,0,1.538042704264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,1,128,1,float16,float16,0,0.7412106990814209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,1,128,1,float16,fp8,0,0.7370453675587972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,8,128,1,float16,fp8,0,1.5784106254577637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,1,128,1,fp8,fp8,0,0.6187680164972941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,24,128,1,float16,float16,0,0.9381120204925537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,24,8,128,1,fp8,fp8,0,1.5105600357055664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,2,128,1,float16,float16,0,0.7479626337687174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,24,128,1,fp8,fp8,0,0.7720320224761963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,2,128,1,float16,fp8,0,0.7400906880696615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,24,128,1,float16,fp8,0,0.9038879871368408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,2,128,1,fp8,fp8,0,0.6573599974314371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,4,128,1,float16,float16,0,0.773856004079183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,4,128,1,float16,fp8,0,0.7772213617960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,4,128,1,fp8,fp8,0,0.7395892937978109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,8,128,1,float16,float16,0,0.7819146315256754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,1,128,1,float16,float16,0,0.384986678759257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,1,128,1,float16,fp8,0,0.41123199462890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,8,128,1,float16,fp8,0,0.7808319727579752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,24,8,128,1,fp8,fp8,0,0.7178773085276285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,24,128,1,float16,float16,0,0.4735786517461141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,1,128,1,fp8,fp8,0,0.32066667079925537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,24,128,1,float16,fp8,0,0.457856019337972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,2,128,1,float16,fp8,0,0.4169546763102214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,2,128,1,float16,float16,0,0.38838398456573486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,2,128,1,fp8,fp8,0,0.3317386706670125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,24,128,1,fp8,fp8,0,0.3919573227564494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,4,128,1,float16,float16,0,0.4004746675491333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,4,128,1,fp8,fp8,0,0.368831992149353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,8,128,1,float16,float16,0,0.4067466656366984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,4,128,1,float16,fp8,0,0.39266133308410645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,24,128,1,float16,float16,0,0.24997333685557047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,8,128,1,fp8,fp8,0,0.36797332763671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,24,8,128,1,float16,fp8,0,0.46832001209259033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,1,128,1,float16,float16,0,0.20463999112447104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,24,128,1,float16,fp8,0,0.24395199616750082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,24,128,1,fp8,fp8,0,0.2111253341039022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,1,128,1,float16,fp8,0,0.20539732774098715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,2,128,1,float16,fp8,0,0.2060693303743998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,1,128,1,fp8,fp8,0,0.15890666842460632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,2,128,1,float16,float16,0,0.20579200983047485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,2,128,1,fp8,fp8,0,0.1657813290754954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,4,128,1,fp8,fp8,0,0.17812265952428183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,4,128,1,float16,float16,0,0.2120800018310547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,8,128,1,float16,float16,0,0.21618666251500449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,8,128,1,float16,fp8,0,0.21445866425832114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,24,128,1,float16,float16,0,0.14652799566586813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,8,128,1,fp8,fp8,0,0.18402133385340372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,24,4,128,1,float16,fp8,0,0.20892800887425741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,24,128,1,float16,fp8,0,0.19936533768971762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,1,128,1,float16,float16,0,0.11063466469446818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,24,128,1,fp8,fp8,0,0.10667733351389568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,1,128,1,fp8,fp8,0,0.08798933029174805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,1,128,1,float16,fp8,0,0.11102933684984843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,2,128,1,float16,float16,0,0.11166933178901672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,2,128,1,fp8,fp8,0,0.08874666690826416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,2,128,1,float16,fp8,0,0.11187199751536052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,4,128,1,float16,float16,0,0.11965333422025044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,4,128,1,float16,fp8,0,0.11406399806340535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,4,128,1,fp8,fp8,0,0.09725333253542583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,8,128,1,float16,float16,0,0.11750933527946472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,1,128,1,float16,float16,0,0.05975999931494395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,24,128,1,fp8,fp8,0,0.07375466823577881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,8,128,1,float16,fp8,0,0.11621333161989848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,24,8,128,1,fp8,fp8,0,0.0986293355623881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,24,128,1,float16,float16,0,0.07232533395290375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,1,128,1,float16,fp8,0,0.06001066664854685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,24,128,1,float16,fp8,0,0.06980800131956737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,1,128,1,fp8,fp8,0,0.04955733319123586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,2,128,1,float16,float16,0,0.06089066465695699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,2,128,1,float16,fp8,0,0.06195199986298879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,4,128,1,float16,fp8,0,0.06264000137646993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,4,128,1,fp8,fp8,0,0.05526400109132131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,2,128,1,fp8,fp8,0,0.050757333636283875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,4,128,1,float16,float16,0,0.06309866905212402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,8,128,1,float16,float16,0,0.0644160012404124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,8,128,1,float16,fp8,0,0.06438933312892914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,24,8,128,1,fp8,fp8,0,0.0565226674079895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,24,128,1,float16,float16,0,0.04030400017897288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,1,128,1,float16,float16,0,0.03586666782697042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,24,128,1,fp8,fp8,0,0.03616533428430557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,1,128,1,float16,fp8,0,0.03638399889071783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,24,128,1,float16,fp8,0,0.039317332208156586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,2,128,1,float16,float16,0,0.0365280012289683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,1,128,1,fp8,fp8,0,0.03160000095764796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,2,128,1,float16,fp8,0,0.03668266783157984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,2,128,1,fp8,fp8,0,0.03172266731659571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,4,128,1,float16,float16,0,0.03731200098991394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,4,128,1,float16,fp8,0,0.03763733307520548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,4,128,1,fp8,fp8,0,0.03359466542800268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,8,128,1,float16,fp8,0,0.03775466730197271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,8,128,1,float16,float16,0,0.0374293327331543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,24,8,128,1,fp8,fp8,0,0.03390933324893316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,1,128,1,float16,float16,0,2.3500426610310874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,1,128,1,fp8,fp8,0,2.0778239568074546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,1,128,1,float16,fp8,0,2.4760533968607583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,2,128,1,fp8,fp8,0,2.1653332710266113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,2,128,1,float16,float16,0,2.3933866818745932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,2,128,1,float16,fp8,0,2.307583967844645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,4,128,1,float16,float16,0,2.5003466606140137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,4,128,1,float16,fp8,0,2.5247039794921875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,1,128,1,float16,float16,0,1.1411999861399333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,4,128,1,fp8,fp8,0,2.3905866940816245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,1,128,1,float16,fp8,0,1.133680025736491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,24,128,1,float16,float16,0,1.570917288462321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,8,128,1,fp8,fp8,0,2.415008068084717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,8,128,1,float16,fp8,0,2.9968268076578775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,24,8,128,1,float16,float16,0,2.496234734853109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,24,128,1,float16,fp8,0,1.5092320442199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,24,128,1,fp8,fp8,0,1.3221226533253987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,1,128,1,fp8,fp8,0,1.0427947044372559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,2,128,1,float16,float16,0,1.1572213172912598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,2,128,1,fp8,fp8,0,1.0787200133005779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,2,128,1,float16,fp8,0,1.1392052968343098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,4,128,1,float16,float16,0,1.3564747174580891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,4,128,1,float16,fp8,0,1.2306026617685955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,4,128,1,fp8,fp8,0,1.2099200089772542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,8,128,1,float16,float16,0,1.2188053131103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,8,128,1,float16,fp8,0,1.2669333616892497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,1,128,1,float16,fp8,0,0.5795573393503824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,1,128,1,float16,float16,0,0.5817866722742716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,24,128,1,float16,float16,0,0.784447987874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,24,128,1,float16,fp8,0,0.7513386408487955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,1,128,1,fp8,fp8,0,0.5131893157958984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,24,8,128,1,fp8,fp8,0,1.214138666788737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,2,128,1,float16,float16,0,0.5888853470484415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,2,128,1,float16,fp8,0,0.5812053283055624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,24,128,1,fp8,fp8,0,0.8474613030751547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,2,128,1,fp8,fp8,0,0.5607039928436279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,4,128,1,float16,float16,0,0.6205973227818807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,4,128,1,float16,fp8,0,0.6070186694463094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,4,128,1,fp8,fp8,0,0.6086346705754598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,1,128,1,float16,float16,0,0.30305065711339313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,8,128,1,float16,fp8,0,0.6181066830952963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,1,128,1,float16,fp8,0,0.30110400915145874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,24,128,1,float16,float16,0,0.5751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,8,128,1,float16,float16,0,0.622650663057963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,24,128,1,float16,fp8,0,0.3837013244628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,24,8,128,1,fp8,fp8,0,0.6190773248672485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,24,128,1,fp8,fp8,0,0.3409279982248942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,1,128,1,fp8,fp8,0,0.26657066742579144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,2,128,1,float16,fp8,0,0.3019040028254191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,2,128,1,fp8,fp8,0,0.278437336285909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,2,128,1,float16,float16,0,0.30532799164454144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,4,128,1,fp8,fp8,0,0.3152800003687541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,4,128,1,float16,fp8,0,0.3115680019060771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,4,128,1,float16,float16,0,0.31734933455785114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,8,128,1,float16,float16,0,0.32475733757019043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,8,128,1,float16,fp8,0,0.31853334108988446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,24,128,1,float16,float16,0,0.29629866282145184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,1,128,1,fp8,fp8,0,0.1325813333193461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,24,8,128,1,fp8,fp8,0,0.31407999992370605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,24,128,1,float16,fp8,0,0.20339200894037882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,1,128,1,float16,float16,0,0.1613759994506836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,24,128,1,fp8,fp8,0,0.18357867002487183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,2,128,1,float16,float16,0,0.1618773341178894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,1,128,1,float16,fp8,0,0.15984533230463663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,2,128,1,float16,fp8,0,0.1798080007235209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,2,128,1,fp8,fp8,0,0.13380266229311624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,4,128,1,float16,float16,0,0.1690559983253479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,4,128,1,float16,fp8,0,0.16593600312868753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,4,128,1,fp8,fp8,0,0.15194666385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,8,128,1,float16,fp8,0,0.17100266615549722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,8,128,1,float16,float16,0,0.17455466588338217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,24,128,1,float16,float16,0,0.18565332889556885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,24,128,1,float16,fp8,0,0.12526933352152506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,1,128,1,float16,float16,0,0.08975999553998311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,24,128,1,fp8,fp8,0,0.09591466188430786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,24,8,128,1,fp8,fp8,0,0.15878933668136597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,1,128,1,fp8,fp8,0,0.07477333148320515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,1,128,1,float16,fp8,0,0.09057066837946574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,2,128,1,float16,fp8,0,0.09467732906341553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,2,128,1,float16,float16,0,0.09220799803733826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,2,128,1,fp8,fp8,0,0.0753119985262553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,4,128,1,float16,float16,0,0.09438400467236836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,4,128,1,float16,fp8,0,0.09402133027712505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,8,128,1,float16,float16,0,0.09707732995351155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,4,128,1,fp8,fp8,0,0.08372267087300618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,24,128,1,float16,float16,0,0.06362133224805196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,24,128,1,float16,fp8,0,0.06101333101590475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,8,128,1,fp8,fp8,0,0.09251200159390767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,24,8,128,1,float16,fp8,0,0.09504000345865886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,1,128,1,float16,float16,0,0.05036800106366476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,24,128,1,fp8,fp8,0,0.0565226674079895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,1,128,1,float16,fp8,0,0.05042133231957754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,2,128,1,float16,fp8,0,0.05087999999523163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,2,128,1,float16,float16,0,0.051818668842315674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,2,128,1,fp8,fp8,0,0.0440533310174942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,4,128,1,float16,fp8,0,0.05297600229581197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,1,128,1,fp8,fp8,0,0.04286933441956838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,4,128,1,fp8,fp8,0,0.048997332652409874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,4,128,1,float16,float16,0,0.05235200126965841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,8,128,1,float16,float16,0,0.05418133238951365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,8,128,1,float16,fp8,0,0.05433600147565206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,24,8,128,1,fp8,fp8,0,0.05004266897837321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,24,128,1,float16,fp8,0,0.03482133398453394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,24,128,1,float16,float16,0,0.03591466695070267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,1,128,1,fp8,fp8,0,0.029605334003766377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,1,128,1,float16,float16,0,0.032170665760835014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,24,128,1,fp8,fp8,0,0.03335466732581457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,1,128,1,float16,fp8,0,0.03180266668399175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,2,128,1,float16,float16,0,0.0322080006202062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,2,128,1,float16,fp8,0,0.03284800052642822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,2,128,1,fp8,fp8,0,0.02940266579389572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,4,128,1,float16,float16,0,0.033359999457995095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,4,128,1,float16,fp8,0,0.03323200096686681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,4,128,1,fp8,fp8,0,0.03120533376932144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,8,128,1,fp8,fp8,0,0.0314026673634847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,8,128,1,float16,float16,0,0.033471999069054924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,24,128,1,float16,float16,0,0.026741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,24,8,128,1,float16,fp8,0,0.03329066683848699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,24,128,1,float16,fp8,0,0.02630399912595749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,1,128,1,float16,float16,0,0.025194667279720306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,24,128,1,fp8,fp8,0,0.02497066557407379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,1,128,1,fp8,fp8,0,0.023024000227451324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,2,128,1,float16,fp8,0,0.02621866762638092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,4,128,1,float16,fp8,0,0.026661333938439686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,1,128,1,float16,fp8,0,0.02548266698916753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,4,128,1,float16,float16,0,0.02629333237806956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,4,128,1,fp8,fp8,0,0.024346667031447094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,2,128,1,fp8,fp8,0,0.023103999594847362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,8,128,1,float16,float16,0,0.026208000878492992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,2,128,1,float16,float16,0,0.025472000241279602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,8,128,1,float16,fp8,0,0.026943999032179516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,24,8,128,1,fp8,fp8,0,0.02462933212518692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,1,128,1,float16,fp8,0,0.9755146503448486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,1,128,1,float16,float16,0,0.9881652990976969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,1,128,1,fp8,fp8,0,0.9050719738006592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,2,128,1,float16,fp8,0,0.9897973537445068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,2,128,1,fp8,fp8,0,0.9006826877593994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,4,128,1,float16,float16,0,1.1104319890340169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,4,128,1,float16,fp8,0,1.083184003829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,2,128,1,float16,float16,0,1.0318506558736165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,1,128,1,float16,float16,0,0.5042453209559122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,4,128,1,fp8,fp8,0,1.0643093585968018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,8,128,1,float16,float16,0,1.0960533618927002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,24,128,1,float16,fp8,0,0.6972586313883463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,8,128,1,fp8,fp8,0,1.0963786443074544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,24,8,128,1,float16,fp8,0,1.3830347061157227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,24,128,1,float16,float16,0,0.726751963297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,24,128,1,fp8,fp8,0,0.5872639815012614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,1,128,1,float16,fp8,0,0.4993973175684611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,1,128,1,fp8,fp8,0,0.4246506690979004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,2,128,1,float16,fp8,0,0.5061386823654175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,2,128,1,float16,float16,0,0.5133119821548462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,2,128,1,fp8,fp8,0,0.4535040060679118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,4,128,1,float16,float16,0,0.6305866638819376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,4,128,1,fp8,fp8,0,0.534437338511149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,4,128,1,float16,fp8,0,0.5306719938913981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,8,128,1,float16,float16,0,0.546234647432963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,1,128,1,float16,float16,0,0.2622026602427165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,8,128,1,float16,fp8,0,0.5483839909235636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,1,128,1,float16,fp8,0,0.26065067450205487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,24,8,128,1,fp8,fp8,0,0.5515146652857462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,24,128,1,float16,float16,0,0.37040531635284424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,24,128,1,fp8,fp8,0,0.41881601015726727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,24,128,1,float16,fp8,0,0.3568106492360433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,1,128,1,fp8,fp8,0,0.22365866104761759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,2,128,1,float16,float16,0,0.2678933342297872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,2,128,1,float16,fp8,0,0.2631839911142985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,2,128,1,fp8,fp8,0,0.2330026626586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,4,128,1,float16,float16,0,0.2809813419977824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,4,128,1,float16,fp8,0,0.2736266652743022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,4,128,1,fp8,fp8,0,0.30722665786743164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,8,128,1,float16,fp8,0,0.2804746627807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,8,128,1,float16,float16,0,0.28384532531102497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,24,8,128,1,fp8,fp8,0,0.2818293372790019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,24,128,1,float16,float16,0,0.19765333334604898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,1,128,1,float16,float16,0,0.1395093301932017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,24,128,1,float16,fp8,0,0.19036267201105753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,1,128,1,float16,fp8,0,0.1585546632607778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,24,128,1,fp8,fp8,0,0.1693120002746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,1,128,1,fp8,fp8,0,0.11864533027013142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,2,128,1,float16,float16,0,0.14251200358072916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,2,128,1,fp8,fp8,0,0.12165866295496623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,2,128,1,float16,fp8,0,0.14079466462135315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,4,128,1,fp8,fp8,0,0.14586666226387024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,4,128,1,float16,fp8,0,0.14618133505185446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,4,128,1,float16,float16,0,0.14847466349601746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,8,128,1,float16,float16,0,0.15402666727701822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,8,128,1,fp8,fp8,0,0.14434132973353067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,24,8,128,1,float16,fp8,0,0.15089600284894308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,1,128,1,float16,float16,0,0.08088000118732452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,24,128,1,float16,float16,0,0.11818666259447734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,24,128,1,float16,fp8,0,0.11352533102035522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,1,128,1,float16,fp8,0,0.08137066662311554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,24,128,1,fp8,fp8,0,0.11998400092124939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,1,128,1,fp8,fp8,0,0.06775466601053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,2,128,1,float16,float16,0,0.08297066887219746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,4,128,1,float16,float16,0,0.08673600355784099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,2,128,1,fp8,fp8,0,0.06921599805355072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,2,128,1,float16,fp8,0,0.0831573357184728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,4,128,1,float16,fp8,0,0.08569066723187764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,4,128,1,fp8,fp8,0,0.07656533519426982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,8,128,1,float16,float16,0,0.10347200433413188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,8,128,1,float16,fp8,0,0.08793600400288899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,24,8,128,1,fp8,fp8,0,0.07760533193747203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,1,128,1,float16,float16,0,0.046165332198143005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,1,128,1,fp8,fp8,0,0.038949333131313324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,24,128,1,float16,fp8,0,0.05835199852784475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,24,128,1,float16,float16,0,0.06032533446947733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,1,128,1,float16,fp8,0,0.0459199994802475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,2,128,1,float16,fp8,0,0.04642133414745331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,2,128,1,float16,float16,0,0.04725866516431173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,24,128,1,fp8,fp8,0,0.0517439991235733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,4,128,1,float16,float16,0,0.049786667029062905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,2,128,1,fp8,fp8,0,0.0394400010506312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,4,128,1,float16,fp8,0,0.047877331574757896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,4,128,1,fp8,fp8,0,0.044351999958356224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,8,128,1,float16,fp8,0,0.04982399940490723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,8,128,1,fp8,fp8,0,0.04558933277924856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,24,8,128,1,float16,float16,0,0.05041599770387014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,24,128,1,float16,fp8,0,0.033157333731651306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,24,128,1,fp8,fp8,0,0.030960001051425934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,1,128,1,float16,float16,0,0.030858665704727173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,1,128,1,float16,fp8,0,0.031114667654037476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,2,128,1,float16,float16,0,0.03126933425664902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,1,128,1,fp8,fp8,0,0.027002667387326557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,24,128,1,float16,float16,0,0.03425599883000056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,2,128,1,fp8,fp8,0,0.027717334528764088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,2,128,1,float16,fp8,0,0.031221332649389904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,4,128,1,float16,float16,0,0.03230933348337809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,4,128,1,float16,fp8,0,0.03221333275238673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,4,128,1,fp8,fp8,0,0.029317334294319153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,8,128,1,float16,float16,0,0.03286933402220408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,8,128,1,float16,fp8,0,0.03268266717592875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,24,128,1,float16,float16,0,0.024165332317352295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,24,128,1,float16,fp8,0,0.024400000770886738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,1,128,1,float16,fp8,0,0.023029332359631855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,1,128,1,fp8,fp8,0,0.02089600016673406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,24,128,1,fp8,fp8,0,0.022848000129063923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,2,128,1,float16,float16,0,0.022997332115968067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,24,8,128,1,fp8,fp8,0,0.029253333806991577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,1,128,1,float16,float16,0,0.02279466638962428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,2,128,1,float16,fp8,0,0.02271999915440877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,2,128,1,fp8,fp8,0,0.02109333376089732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,8,128,1,float16,float16,0,0.023525332411130268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,4,128,1,float16,float16,0,0.02367999901374181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,4,128,1,fp8,fp8,0,0.022175999979178112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,4,128,1,float16,fp8,0,0.023760000864664715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,24,128,1,float16,float16,0,0.019685332973798115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,8,128,1,fp8,fp8,0,0.02232533444960912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,24,8,128,1,float16,fp8,0,0.024154665569464367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,24,128,1,float16,fp8,0,0.019760000209013622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,2,128,1,float16,float16,0,0.018992000569899876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,24,128,1,fp8,fp8,0,0.018378666291634243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,1,128,1,float16,float16,0,0.018842666099468868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,1,128,1,float16,fp8,0,0.018981333822011948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,1,128,1,fp8,fp8,0,0.017797333498795826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,2,128,1,float16,fp8,0,0.01894933357834816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,4,128,1,float16,float16,0,0.018725333114465077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,4,128,1,float16,fp8,0,0.01966399947802226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,4,128,1,fp8,fp8,0,0.017893332988023758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,8,128,1,float16,float16,0,0.019029332945744198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,8,128,1,float16,fp8,0,0.01939733326435089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,8,128,1,fp8,fp8,0,0.018112000077962875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,24,2,128,1,fp8,fp8,0,0.017605333278576534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,1,128,1,float16,float16,0,0.46613868077596027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,1,128,1,fp8,fp8,0,0.46590399742126465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,2,128,1,float16,float16,0,0.4774826765060425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,2,128,1,float16,fp8,0,0.47102399667104083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,2,128,1,fp8,fp8,0,0.46054399013519287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,4,128,1,float16,float16,0,0.5195786555608114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,1,128,1,float16,fp8,0,0.4643253485361735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,4,128,1,float16,fp8,0,0.5020906527837118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,4,128,1,fp8,fp8,0,0.5318400065104166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,8,128,1,float16,float16,0,0.7196533679962158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,1,128,1,float16,float16,0,0.24443199237187704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,24,128,1,float16,float16,0,0.38475199540456134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,24,128,1,fp8,fp8,0,0.2983466585477193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,8,128,1,float16,fp8,0,0.5140320062637329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,24,128,1,float16,fp8,0,0.3761013348897298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,24,8,128,1,fp8,fp8,0,0.5498346487681071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,1,128,1,float16,fp8,0,0.24568533897399902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,1,128,1,fp8,fp8,0,0.24333866437276205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,2,128,1,float16,fp8,0,0.24898666143417358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,2,128,1,float16,float16,0,0.25146132707595825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,2,128,1,fp8,fp8,0,0.23504533370335898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,4,128,1,float16,float16,0,0.26581867535909015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,4,128,1,float16,fp8,0,0.2611733277638753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,4,128,1,fp8,fp8,0,0.2747039993604024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,8,128,1,float16,float16,0,0.27305599053700763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,1,128,1,float16,float16,0,0.13307199875513712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,1,128,1,float16,fp8,0,0.13345066706339517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,8,128,1,float16,fp8,0,0.3616960048675537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,24,128,1,float16,fp8,0,0.19502399365107217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,24,128,1,fp8,fp8,0,0.15945600469907126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,24,8,128,1,fp8,fp8,0,0.2820640007654826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,1,128,1,fp8,fp8,0,0.11916800340016682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,24,128,1,float16,float16,0,0.2050186594327291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,2,128,1,float16,float16,0,0.13648533821105957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,2,128,1,float16,fp8,0,0.1348479986190796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,2,128,1,fp8,fp8,0,0.12894399960835776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,4,128,1,float16,fp8,0,0.14099199573198953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,4,128,1,fp8,fp8,0,0.13769599795341492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,4,128,1,float16,float16,0,0.1439786652723948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,8,128,1,float16,float16,0,0.1493119994799296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,8,128,1,fp8,fp8,0,0.14247467120488486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,24,128,1,float16,float16,0,0.109525332848231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,24,128,1,float16,fp8,0,0.14536533753077188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,24,8,128,1,float16,fp8,0,0.14648000399271646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,1,128,1,float16,float16,0,0.07566399872303009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,1,128,1,float16,fp8,0,0.07592533528804779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,1,128,1,fp8,fp8,0,0.06727466483910878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,24,128,1,fp8,fp8,0,0.08859200278917949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,2,128,1,float16,float16,0,0.07764266431331635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,2,128,1,float16,fp8,0,0.07655466596285503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,2,128,1,fp8,fp8,0,0.0728053351243337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,4,128,1,float16,fp8,0,0.08110400040944417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,4,128,1,float16,float16,0,0.08172800143559773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,8,128,1,float16,fp8,0,0.08243733147780101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,4,128,1,fp8,fp8,0,0.07598933577537537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,8,128,1,fp8,fp8,0,0.07696533203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,24,128,1,float16,float16,0,0.0598880002895991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,24,8,128,1,float16,float16,0,0.08435199658075969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,24,128,1,float16,fp8,0,0.0709440012772878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,1,128,1,float16,float16,0,0.043840001026789345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,24,128,1,fp8,fp8,0,0.05093866586685181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,1,128,1,float16,fp8,0,0.043050666650136314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,2,128,1,float16,fp8,0,0.04484800000985464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,2,128,1,float16,float16,0,0.0447626660267512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,1,128,1,fp8,fp8,0,0.038346665600935616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,4,128,1,float16,float16,0,0.04703466594219208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,2,128,1,fp8,fp8,0,0.039162665605545044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,4,128,1,fp8,fp8,0,0.043680002291997276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,8,128,1,float16,float16,0,0.04770666857560476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,4,128,1,float16,fp8,0,0.04675200084845225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,8,128,1,float16,fp8,0,0.04797866443792979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,24,128,1,float16,fp8,0,0.03314133236805598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,24,8,128,1,fp8,fp8,0,0.045125335454940796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,1,128,1,float16,float16,0,0.03054933249950409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,1,128,1,float16,fp8,0,0.02998399982849757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,24,128,1,fp8,fp8,0,0.031583999594052635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,2,128,1,float16,float16,0,0.03054400036732356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,2,128,1,float16,fp8,0,0.03075733284155528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,1,128,1,fp8,fp8,0,0.027050666511058807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,24,128,1,float16,float16,0,0.03233599911133448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,4,128,1,float16,float16,0,0.031701333820819855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,2,128,1,fp8,fp8,0,0.0276853342851003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,4,128,1,float16,fp8,0,0.03179199993610382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,8,128,1,float16,fp8,0,0.03190933416287104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,8,128,1,float16,float16,0,0.03212266663710276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,4,128,1,fp8,fp8,0,0.02924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,24,8,128,1,fp8,fp8,0,0.02902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,24,128,1,float16,fp8,0,0.023823998868465424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,24,128,1,fp8,fp8,0,0.022645334402720135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,1,128,1,float16,float16,0,0.021301334102948506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,1,128,1,float16,fp8,0,0.021946666141351063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,24,128,1,float16,float16,0,0.023024000227451324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,1,128,1,fp8,fp8,0,0.020794666061798733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,2,128,1,float16,float16,0,0.021717332303524017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,2,128,1,fp8,fp8,0,0.021418665846188862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,2,128,1,float16,fp8,0,0.021829334398110706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,4,128,1,float16,fp8,0,0.02237333357334137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,4,128,1,fp8,fp8,0,0.021840001145998638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,4,128,1,float16,float16,0,0.022245332598686218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,8,128,1,float16,fp8,0,0.022634667654832203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,8,128,1,fp8,fp8,0,0.022522665560245514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,24,8,128,1,float16,float16,0,0.022453332940737408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,24,128,1,float16,float16,0,0.018079999834299088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,1,128,1,float16,float16,0,0.017360000560681026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,24,128,1,fp8,fp8,0,0.01829333355029424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,1,128,1,float16,fp8,0,0.01775466650724411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,1,128,1,fp8,fp8,0,0.01766933376590411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,2,128,1,float16,float16,0,0.017973333597183228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,24,128,1,float16,fp8,0,0.023007998863856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,4,128,1,float16,float16,0,0.017759999881188076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,2,128,1,float16,fp8,0,0.01791999985774358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,2,128,1,fp8,fp8,0,0.01786133274435997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,4,128,1,fp8,fp8,0,0.017845333864291508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,4,128,1,float16,fp8,0,0.018144000321626663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,8,128,1,float16,float16,0,0.017968000223239262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,8,128,1,float16,fp8,0,0.018325333793958027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,24,128,1,float16,float16,0,0.017029333859682083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,1,128,1,float16,float16,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,24,128,1,fp8,fp8,0,0.017071999609470367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,1,128,1,float16,fp8,0,0.017360000560681026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,1,128,1,fp8,fp8,0,0.017125333348910015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,24,8,128,1,fp8,fp8,0,0.018122666825850803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,24,128,1,float16,fp8,0,0.017024000485738117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,2,128,1,float16,float16,0,0.017114666601022083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,2,128,1,float16,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,2,128,1,fp8,fp8,0,0.016837333639462788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,4,128,1,float16,float16,0,0.01722666621208191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,4,128,1,float16,fp8,0,0.017322666943073273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,4,128,1,fp8,fp8,0,0.017130666722853977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,8,128,1,float16,float16,0,0.01699200024207433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,8,128,1,fp8,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,24,8,128,1,float16,fp8,0,0.01701333373785019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,1,128,1,float16,fp8,0,0.2884320020675659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,1,128,1,float16,float16,0,0.33979201316833496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,1,128,1,fp8,fp8,0,0.29446399211883545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,2,128,1,float16,float16,0,0.29364800453186035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,2,128,1,float16,fp8,0,0.29020800193150836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,2,128,1,fp8,fp8,0,0.3171360095342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,4,128,1,float16,float16,0,0.3070879975954692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,4,128,1,float16,fp8,0,0.30221333106358844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,4,128,1,fp8,fp8,0,0.34327467282613117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,8,128,1,float16,float16,0,0.410586675008138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,8,128,1,float16,fp8,0,0.3114773432413737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,24,128,1,float16,fp8,0,0.19340266784032187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,24,8,128,1,fp8,fp8,0,0.34778134028116864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,24,128,1,float16,float16,0,0.2001333236694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,24,128,1,fp8,fp8,0,0.19386667013168335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,1,128,1,float16,float16,0,0.15127999583880106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,1,128,1,float16,fp8,0,0.15264532963434854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,1,128,1,fp8,fp8,0,0.16404267152150473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,2,128,1,float16,fp8,0,0.1551253298918406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,2,128,1,float16,float16,0,0.15486933787663779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,2,128,1,fp8,fp8,0,0.1607093314329783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,4,128,1,float16,fp8,0,0.16033066312472025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,4,128,1,float16,float16,0,0.1630346675713857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,4,128,1,fp8,fp8,0,0.17554134130477905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,1,128,1,float16,float16,0,0.08396266897519429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,24,128,1,float16,float16,0,0.10706667105356853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,8,128,1,float16,float16,0,0.1699999968210856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,24,128,1,float16,fp8,0,0.10248000423113506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,8,128,1,float16,fp8,0,0.21259733041127524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,24,8,128,1,fp8,fp8,0,0.18196799357732138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,24,128,1,fp8,fp8,0,0.10682132840156555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,1,128,1,float16,fp8,0,0.08404800295829773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,1,128,1,fp8,fp8,0,0.08727999528249104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,2,128,1,float16,float16,0,0.08601599931716919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,2,128,1,float16,fp8,0,0.08560533324877422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,2,128,1,fp8,fp8,0,0.08998933434486389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,4,128,1,float16,float16,0,0.09474666913350423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,4,128,1,float16,fp8,0,0.0883786678314209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,4,128,1,fp8,fp8,0,0.09715200463930766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,8,128,1,float16,fp8,0,0.09149332841237386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,1,128,1,float16,float16,0,0.046911999583244324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,8,128,1,float16,float16,0,0.09223467111587524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,24,8,128,1,fp8,fp8,0,0.09760000308354695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,24,128,1,fp8,fp8,0,0.0713973343372345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,24,128,1,float16,fp8,0,0.05691733459631602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,24,128,1,float16,float16,0,0.05871999760468801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,1,128,1,float16,fp8,0,0.04761599997679392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,1,128,1,fp8,fp8,0,0.04885333279768626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,2,128,1,float16,float16,0,0.04790933430194855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,4,128,1,float16,float16,0,0.05183466772238413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,2,128,1,float16,fp8,0,0.048357332746187844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,2,128,1,fp8,fp8,0,0.05017599960168203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,4,128,1,float16,fp8,0,0.050426666935284935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,4,128,1,fp8,fp8,0,0.05436799923578898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,8,128,1,float16,float16,0,0.05186133086681366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,8,128,1,float16,fp8,0,0.05170666674772898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,24,8,128,1,fp8,fp8,0,0.055770665407180786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,1,128,1,float16,float16,0,0.029765332738558452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,24,128,1,float16,float16,0,0.03178666780392329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,24,128,1,fp8,fp8,0,0.03356799980004629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,24,128,1,float16,fp8,0,0.031034665803114574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,1,128,1,fp8,fp8,0,0.030858665704727173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,1,128,1,float16,fp8,0,0.030048000315825146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,2,128,1,float16,float16,0,0.03028800090154012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,2,128,1,float16,fp8,0,0.03029866764942805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,4,128,1,float16,float16,0,0.030928000807762146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,4,128,1,float16,fp8,0,0.03136000037193298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,2,128,1,fp8,fp8,0,0.031119999786218006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,4,128,1,fp8,fp8,0,0.03297599901755651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,8,128,1,float16,float16,0,0.031317333380381264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,8,128,1,float16,fp8,0,0.03189333279927572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,24,8,128,1,fp8,fp8,0,0.03301866600910822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,24,128,1,float16,fp8,0,0.02476266771554947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,1,128,1,float16,float16,0,0.023738667368888855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,24,128,1,fp8,fp8,0,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,2,128,1,fp8,fp8,0,0.02447466552257538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,24,128,1,float16,float16,0,0.02426133304834366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,1,128,1,fp8,fp8,0,0.024298667907714844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,2,128,1,float16,float16,0,0.023530667026837666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,2,128,1,float16,fp8,0,0.02372266600529353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,1,128,1,float16,fp8,0,0.023573334018389385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,4,128,1,float16,float16,0,0.02420799930890401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,4,128,1,float16,fp8,0,0.024703999360402424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,4,128,1,fp8,fp8,0,0.025445332129796345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,8,128,1,float16,fp8,0,0.024346667031447094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,24,128,1,float16,float16,0,0.0179626668492953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,8,128,1,fp8,fp8,0,0.025834667185942333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,24,8,128,1,float16,float16,0,0.02491733431816101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,1,128,1,float16,float16,0,0.01676799977819125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,1,128,1,float16,fp8,0,0.016800000021855038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,24,128,1,float16,fp8,0,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,24,128,1,fp8,fp8,0,0.01842133328318596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,1,128,1,fp8,fp8,0,0.01757866640885671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,2,128,1,float16,float16,0,0.017008000363906223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,2,128,1,float16,fp8,0,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,4,128,1,float16,fp8,0,0.016805333395799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,2,128,1,fp8,fp8,0,0.017535999417304993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,4,128,1,float16,float16,0,0.016869333883126576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,8,128,1,float16,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,8,128,1,float16,float16,0,0.01684800038735072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,8,128,1,fp8,fp8,0,0.01855466639002164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,24,128,1,float16,fp8,0,0.01607999950647354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,24,4,128,1,fp8,fp8,0,0.019989332805077236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,24,128,1,float16,float16,0,0.015615999698638916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,24,128,1,fp8,fp8,0,0.017082666357358296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,1,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,1,128,1,float16,float16,0,0.015765332927306492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,2,128,1,float16,fp8,0,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,1,128,1,fp8,fp8,0,0.016821333517630894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,2,128,1,float16,float16,0,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,2,128,1,fp8,fp8,0,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,4,128,1,fp8,fp8,0,0.017077332983414333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,8,128,1,float16,fp8,0,0.016197333733240765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,4,128,1,float16,float16,0,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,4,128,1,float16,fp8,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,8,128,1,float16,float16,0,0.015909332782030106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,24,8,128,1,fp8,fp8,0,0.01725333308180173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,24,128,1,float16,float16,0,0.015210667004187902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,1,128,1,float16,float16,0,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,24,128,1,fp8,fp8,0,0.0163680004576842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,1,128,1,float16,fp8,0,0.01543466622630755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,2,128,1,float16,float16,0,0.015397333850463232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,2,128,1,float16,fp8,0,0.015824000040690105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,1,128,1,fp8,fp8,0,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,2,128,1,fp8,fp8,0,0.016549333930015564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,24,128,1,float16,fp8,0,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,4,128,1,float16,float16,0,0.015200000256299973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,4,128,1,float16,fp8,0,0.01586666703224182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,8,128,1,float16,float16,0,0.015247999380032221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,4,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,8,128,1,float16,fp8,0,0.01581866666674614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,24,8,128,1,fp8,fp8,0,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,1,128,1,fp8,fp8,0,0.24441067377726236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,1,128,1,float16,float16,0,0.20272533098856607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,2,128,1,fp8,fp8,0,0.23987199862798056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,4,128,1,float16,float16,0,0.212501327196757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,2,128,1,float16,float16,0,0.20502400398254395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,2,128,1,float16,fp8,0,0.20410666863123575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,1,128,1,float16,fp8,0,0.20141865809758505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,4,128,1,float16,fp8,0,0.2095306714375814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,4,128,1,fp8,fp8,0,0.2589600086212158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,24,128,1,float16,float16,0,0.12808000048001608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,8,128,1,fp8,fp8,0,0.25989333788553876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,8,128,1,float16,fp8,0,0.216048002243042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,24,8,128,1,float16,float16,0,0.21783467133839926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,1,128,1,float16,float16,0,0.10633066296577454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,24,128,1,float16,fp8,0,0.1239466667175293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,1,128,1,float16,fp8,0,0.1086346705754598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,24,128,1,fp8,fp8,0,0.14503467082977295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,1,128,1,fp8,fp8,0,0.13045333822568259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,2,128,1,float16,float16,0,0.10805333654085796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,2,128,1,fp8,fp8,0,0.13060266772905985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,2,128,1,float16,fp8,0,0.10840533177057902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,4,128,1,float16,fp8,0,0.1128053367137909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,4,128,1,float16,float16,0,0.1125866671403249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,4,128,1,fp8,fp8,0,0.14089066783587137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,8,128,1,float16,float16,0,0.11688533425331116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,8,128,1,float16,fp8,0,0.11523733536402385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,24,8,128,1,fp8,fp8,0,0.13936000068982443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,24,128,1,float16,float16,0,0.07004799942175548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,24,128,1,fp8,fp8,0,0.0821973333756129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,24,128,1,float16,fp8,0,0.06735999882221222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,1,128,1,float16,fp8,0,0.058687999844551086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,1,128,1,float16,float16,0,0.05815466741720835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,1,128,1,fp8,fp8,0,0.06960000097751617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,2,128,1,float16,float16,0,0.059104000528653465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,4,128,1,float16,float16,0,0.06112533311049143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,4,128,1,float16,fp8,0,0.06181333462397257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,2,128,1,float16,fp8,0,0.058693334460258484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,4,128,1,fp8,fp8,0,0.0757120003302892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,2,128,1,fp8,fp8,0,0.07088000078996022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,1,128,1,float16,float16,0,0.034517332911491394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,24,128,1,float16,float16,0,0.03661333272854487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,24,128,1,float16,fp8,0,0.036101333796978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,8,128,1,float16,fp8,0,0.06197333335876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,24,128,1,fp8,fp8,0,0.04384533564249674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,8,128,1,float16,float16,0,0.062458669145902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,1,128,1,float16,fp8,0,0.034901333351929985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,24,8,128,1,fp8,fp8,0,0.07656000057856242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,1,128,1,fp8,fp8,0,0.040805332362651825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,2,128,1,float16,fp8,0,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,2,128,1,float16,float16,0,0.03527999917666117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,4,128,1,float16,float16,0,0.035877334574858345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,4,128,1,fp8,fp8,0,0.04313066601753235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,8,128,1,float16,fp8,0,0.03585600107908249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,8,128,1,float16,float16,0,0.035690667728583016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,4,128,1,float16,fp8,0,0.03595199932654699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,2,128,1,fp8,fp8,0,0.04123199979464213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,1,128,1,float16,float16,0,0.02388266722361247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,24,8,128,1,fp8,fp8,0,0.043178667624791466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,24,128,1,float16,fp8,0,0.025418666501839954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,24,128,1,float16,float16,0,0.025621332228183746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,24,128,1,fp8,fp8,0,0.029338667790095013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,1,128,1,float16,fp8,0,0.023999998966852825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,1,128,1,fp8,fp8,0,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,2,128,1,float16,fp8,0,0.024383999407291412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,4,128,1,float16,float16,0,0.024671999116738636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,4,128,1,fp8,fp8,0,0.02922133356332779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,4,128,1,float16,fp8,0,0.0249493345618248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,2,128,1,fp8,fp8,0,0.02805333336194356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,2,128,1,float16,float16,0,0.024304000039895374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,8,128,1,float16,float16,0,0.024634666740894318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,8,128,1,fp8,fp8,0,0.029301332930723827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,24,8,128,1,float16,fp8,0,0.024800000091393787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,24,128,1,float16,float16,0,0.019071999937295914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,24,128,1,float16,fp8,0,0.019253333409627277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,2,128,1,float16,float16,0,0.018863999595244724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,24,128,1,fp8,fp8,0,0.021722666919231415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,1,128,1,fp8,fp8,0,0.021082667013009388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,1,128,1,float16,fp8,0,0.02163733293612798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,4,128,1,float16,fp8,0,0.018816000471512478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,2,128,1,float16,fp8,0,0.019023999571800232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,2,128,1,fp8,fp8,0,0.021136000752449036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,4,128,1,float16,float16,0,0.018618666877349217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,1,128,1,float16,float16,0,0.018911999960740406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,4,128,1,fp8,fp8,0,0.021082667013009388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,8,128,1,float16,float16,0,0.018559999763965607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,8,128,1,fp8,fp8,0,0.021536000072956085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,24,128,1,float16,fp8,0,0.015775999675194424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,24,8,128,1,float16,fp8,0,0.018826667219400406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,24,128,1,fp8,fp8,0,0.017792000124851864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,1,128,1,float16,fp8,0,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,2,128,1,float16,float16,0,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,24,128,1,float16,float16,0,0.015429332852363586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,1,128,1,float16,float16,0,0.014959999670584997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,2,128,1,float16,fp8,0,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,4,128,1,float16,float16,0,0.015205333630243937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,4,128,1,float16,fp8,0,0.015642666568358738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,1,128,1,fp8,fp8,0,0.017125333348910015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,2,128,1,fp8,fp8,0,0.01701333373785019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,4,128,1,fp8,fp8,0,0.017664000391960144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,8,128,1,fp8,fp8,0,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,24,128,1,float16,float16,0,0.014618666221698126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,24,128,1,float16,fp8,0,0.015279999623696009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,8,128,1,float16,float16,0,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,24,8,128,1,float16,fp8,0,0.01569066693385442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,1,128,1,float16,float16,0,0.01451733335852623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,1,128,1,float16,fp8,0,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,1,128,1,fp8,fp8,0,0.016565332810084026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,2,128,1,float16,float16,0,0.014826666563749313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,2,128,1,float16,fp8,0,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,24,128,1,fp8,fp8,0,0.016634666671355564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,2,128,1,fp8,fp8,0,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,4,128,1,float16,fp8,0,0.014885333677132925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,4,128,1,fp8,fp8,0,0.016890666137139004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,4,128,1,float16,float16,0,0.01441066712141037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,8,128,1,float16,fp8,0,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,8,128,1,fp8,fp8,0,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,24,8,128,1,float16,float16,0,0.014522666732470194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,1,128,1,float16,float16,0,0.014357333381970724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,24,128,1,float16,fp8,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,24,128,1,float16,float16,0,0.014229333649079004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,2,128,1,float16,float16,0,0.0144213338692983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,1,128,1,fp8,fp8,0,0.01611199975013733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,24,128,1,fp8,fp8,0,0.01602666700879733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,2,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,2,128,1,fp8,fp8,0,0.0163680004576842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,1,128,1,float16,fp8,0,0.014474666366974512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,4,128,1,fp8,fp8,0,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,8,128,1,float16,float16,0,0.014357333381970724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,8,128,1,fp8,fp8,0,0.016255999604860943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,4,128,1,float16,float16,0,0.014186666657527288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,4,128,1,float16,fp8,0,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,24,8,128,1,float16,fp8,0,0.014453332871198654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,1,128,1,float16,float16,0,0.17178666591644287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,1,128,1,fp8,fp8,0,0.20549333095550537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,2,128,1,float16,fp8,0,0.17241066694259644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,1,128,1,float16,fp8,0,0.17160000403722128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,2,128,1,float16,float16,0,0.17185600598653158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,2,128,1,fp8,fp8,0,0.20801067352294922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,4,128,1,float16,float16,0,0.17592533429463705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,4,128,1,float16,fp8,0,0.17547732591629028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,4,128,1,fp8,fp8,0,0.21661333243052164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,8,128,1,float16,float16,0,0.17863466342290243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,8,128,1,float16,fp8,0,0.17748266458511353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,1,128,1,float16,fp8,0,0.09131733576456706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,24,128,1,float16,float16,0,0.0958666702111562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,24,8,128,1,fp8,fp8,0,0.2146773338317871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,24,128,1,fp8,fp8,0,0.12157866358757019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,1,128,1,float16,float16,0,0.09056533376375835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,24,128,1,float16,fp8,0,0.09264533718427022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,1,128,1,fp8,fp8,0,0.11032000184059143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,2,128,1,float16,float16,0,0.09148800373077393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,2,128,1,float16,fp8,0,0.0922933320204417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,2,128,1,fp8,fp8,0,0.11110400160153706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,4,128,1,float16,fp8,0,0.0937600036462148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,4,128,1,fp8,fp8,0,0.11542933185895284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,4,128,1,float16,float16,0,0.093941330909729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,8,128,1,float16,float16,0,0.09403199950853984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,8,128,1,float16,fp8,0,0.09402133027712505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,24,128,1,float16,float16,0,0.05020800232887268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,24,8,128,1,fp8,fp8,0,0.11619200309117635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,24,128,1,float16,fp8,0,0.049135997891426086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,24,128,1,fp8,fp8,0,0.06526933113733928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,1,128,1,float16,float16,0,0.05123733480771383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,2,128,1,float16,fp8,0,0.05239466826121012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,1,128,1,float16,fp8,0,0.052341332038243614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,2,128,1,fp8,fp8,0,0.06251200040181477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,2,128,1,float16,float16,0,0.052069331208864846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,4,128,1,float16,float16,0,0.05277866621812185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,4,128,1,float16,fp8,0,0.053114667534828186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,1,128,1,fp8,fp8,0,0.062133332093556724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,4,128,1,fp8,fp8,0,0.06471466521422069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,8,128,1,float16,float16,0,0.05299733579158783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,24,128,1,float16,fp8,0,0.032218667368094124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,8,128,1,fp8,fp8,0,0.06460799773534139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,24,128,1,fp8,fp8,0,0.03959999978542328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,24,128,1,float16,float16,0,0.03226666649182638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,1,128,1,float16,float16,0,0.03276800115903219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,1,128,1,float16,fp8,0,0.03242133309443792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,1,128,1,fp8,fp8,0,0.0383146678407987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,24,8,128,1,float16,fp8,0,0.05385066568851471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,2,128,1,float16,float16,0,0.032511999209721885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,2,128,1,float16,fp8,0,0.03257599969704946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,4,128,1,float16,float16,0,0.03334933271010717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,4,128,1,float16,fp8,0,0.03327466547489166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,2,128,1,fp8,fp8,0,0.03888533264398575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,8,128,1,float16,float16,0,0.03373866776625315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,4,128,1,fp8,fp8,0,0.03995199998219808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,8,128,1,fp8,fp8,0,0.03955733279387156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,24,128,1,float16,float16,0,0.021456000705560047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,24,128,1,float16,fp8,0,0.02216000109910965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,24,8,128,1,float16,fp8,0,0.033941333492596946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,24,128,1,fp8,fp8,0,0.025306666890780132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,1,128,1,float16,fp8,0,0.021183999876181286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,1,128,1,float16,float16,0,0.021125334004561108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,2,128,1,float16,float16,0,0.02123733361562093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,1,128,1,fp8,fp8,0,0.025087999800841015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,2,128,1,float16,fp8,0,0.021312000850836437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,2,128,1,fp8,fp8,0,0.025114665428797405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,4,128,1,float16,fp8,0,0.021407999098300934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,4,128,1,fp8,fp8,0,0.025194667279720306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,4,128,1,float16,float16,0,0.021189334491888683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,24,128,1,float16,float16,0,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,24,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,8,128,1,fp8,fp8,0,0.025546667476495106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,8,128,1,float16,float16,0,0.021407999098300934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,24,8,128,1,float16,fp8,0,0.021525333325068157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,1,128,1,float16,float16,0,0.017850667238235474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,24,128,1,fp8,fp8,0,0.02092266579469045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,1,128,1,fp8,fp8,0,0.02089600016673406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,2,128,1,float16,float16,0,0.017957333475351334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,4,128,1,float16,fp8,0,0.01825599993268649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,2,128,1,fp8,fp8,0,0.020949333906173706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,2,128,1,float16,fp8,0,0.018101333330074947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,1,128,1,float16,fp8,0,0.018133333573738735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,4,128,1,float16,float16,0,0.017984000345071156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,4,128,1,fp8,fp8,0,0.020954666038354237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,8,128,1,float16,float16,0,0.01809599995613098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,8,128,1,float16,fp8,0,0.01844800015290578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,24,8,128,1,fp8,fp8,0,0.021589333812395733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,1,128,1,float16,fp8,0,0.014853333433469137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,24,128,1,float16,fp8,0,0.014864000181357065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,1,128,1,float16,float16,0,0.014463999619086584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,24,128,1,float16,float16,0,0.014815999815861383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,24,128,1,fp8,fp8,0,0.016757333030303318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,1,128,1,fp8,fp8,0,0.01682666689157486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,2,128,1,float16,float16,0,0.01463466634353002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,2,128,1,float16,fp8,0,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,2,128,1,fp8,fp8,0,0.016906666258970898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,4,128,1,float16,fp8,0,0.015295999745527903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,8,128,1,float16,fp8,0,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,8,128,1,fp8,fp8,0,0.01730666682124138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,8,128,1,float16,float16,0,0.01471466695268949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,4,128,1,fp8,fp8,0,0.017162666966517765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,24,4,128,1,float16,float16,0,0.0143306665122509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,24,128,1,float16,fp8,0,0.01481066644191742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,24,128,1,float16,float16,0,0.014490666488806406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,1,128,1,float16,float16,0,0.014197333405415217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,24,128,1,fp8,fp8,0,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,1,128,1,float16,fp8,0,0.01479999969402949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,1,128,1,fp8,fp8,0,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,2,128,1,float16,float16,0,0.01434133326013883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,2,128,1,float16,fp8,0,0.01451733335852623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,2,128,1,fp8,fp8,0,0.016666666915019352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,4,128,1,float16,float16,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,4,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,4,128,1,fp8,fp8,0,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,8,128,1,float16,float16,0,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,24,128,1,float16,fp8,0,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,8,128,1,fp8,fp8,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,24,128,1,float16,float16,0,0.013904000322024027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,1,128,1,float16,float16,0,0.014111999422311783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,24,8,128,1,float16,fp8,0,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,1,128,1,float16,fp8,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,24,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,1,128,1,fp8,fp8,0,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,2,128,1,float16,float16,0,0.013967999567588171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,2,128,1,fp8,fp8,0,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,4,128,1,float16,float16,0,0.014127999544143677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,4,128,1,fp8,fp8,0,0.016021333634853363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,2,128,1,float16,fp8,0,0.014639999717473984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,8,128,1,float16,fp8,0,0.014533333480358124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,4,128,1,float16,fp8,0,0.014666666587193808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,8,128,1,float16,float16,0,0.014170666535695394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,24,8,128,1,fp8,fp8,0,0.01618133361140887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,1,128,1,float16,fp8,0,16.153701782226562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,1,128,1,fp8,fp8,0,10.534725189208984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,1,128,1,float16,float16,0,16.693200429280598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,2,128,1,fp8,fp8,0,10.571397145589193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,2,128,1,float16,float16,0,16.48370615641276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,2,128,1,float16,fp8,0,16.212331136067707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,4,128,1,float16,fp8,0,16.49178695678711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,4,128,1,float16,float16,0,16.615824381510418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,1,128,1,float16,float16,0,8.22319475809733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,4,128,1,fp8,fp8,0,10.632816314697266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,16,128,1,float16,float16,0,8.05893325805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,1,128,1,float16,fp8,0,8.311354955037435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,16,128,1,float16,fp8,0,7.973728179931641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,8,128,1,float16,float16,0,16.53806432088216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,16,128,1,fp8,fp8,0,5.271077473958333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,1,128,1,fp8,fp8,0,5.108160018920898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,8,128,1,float16,fp8,0,16.71986134847005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,16,8,128,1,fp8,fp8,0,10.703819274902344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,2,128,1,float16,float16,0,8.254757563273111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,2,128,1,float16,fp8,0,8.272160212198893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,2,128,1,fp8,fp8,0,5.295445442199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,4,128,1,fp8,fp8,0,5.2491200764973955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,4,128,1,float16,fp8,0,8.178576151529947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,4,128,1,float16,float16,0,8.286848068237305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,8,128,1,float16,float16,0,8.465861638387045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,1,128,1,float16,float16,0,3.978480021158854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,16,128,1,float16,float16,0,4.012618700663249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,1,128,1,float16,fp8,0,4.010085423787435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,8,128,1,fp8,fp8,0,5.173178672790527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,16,8,128,1,float16,fp8,0,8.146490732828775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,1,128,1,fp8,fp8,0,2.676608085632324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,16,128,1,float16,fp8,0,3.9952265421549478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,16,128,1,fp8,fp8,0,2.6037599245707193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,2,128,1,float16,float16,0,3.9865493774414062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,2,128,1,float16,fp8,0,4.076239903767903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,2,128,1,fp8,fp8,0,2.6586292584737143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,4,128,1,float16,float16,0,3.946880022684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,4,128,1,float16,fp8,0,3.9295838673909507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,4,128,1,fp8,fp8,0,2.687679926554362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,8,128,1,float16,float16,0,4.136368115743001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,16,128,1,float16,float16,0,2.0658559799194336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,1,128,1,float16,float16,0,2.084906737009684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,1,128,1,float16,fp8,0,2.1101226806640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,8,128,1,fp8,fp8,0,2.680837313334147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,16,8,128,1,float16,fp8,0,4.007487932840983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,16,128,1,fp8,fp8,0,1.395583947499593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,1,128,1,fp8,fp8,0,1.4366186459859211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,16,128,1,float16,fp8,0,2.0362186431884766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,2,128,1,float16,float16,0,2.100693384806315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,2,128,1,fp8,fp8,0,1.4330612818400066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,2,128,1,float16,fp8,0,2.070261319478353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,4,128,1,float16,fp8,0,2.0734240214029946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,4,128,1,float16,float16,0,2.1003947257995605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,4,128,1,fp8,fp8,0,1.43341859181722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,8,128,1,float16,float16,0,2.103936036427816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,8,128,1,fp8,fp8,0,1.4545599619547527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,16,8,128,1,float16,fp8,0,2.0903627077738443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,1,128,1,float16,float16,0,9.742666880289713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,1,128,1,fp8,fp8,0,6.119199752807617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,1,128,1,float16,fp8,0,9.58244260152181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,2,128,1,float16,fp8,0,9.407573064168295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,2,128,1,float16,float16,0,9.52566401163737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,2,128,1,fp8,fp8,0,6.157530466715495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,4,128,1,float16,float16,0,9.781509399414062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,4,128,1,float16,fp8,0,9.55624008178711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,4,128,1,fp8,fp8,0,6.1752268473307295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,1,128,1,float16,float16,0,4.723573366800944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,16,128,1,float16,float16,0,4.632346789042155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,16,128,1,float16,fp8,0,4.636058807373047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,8,128,1,float16,float16,0,9.626821517944336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,8,128,1,fp8,fp8,0,6.165626525878906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,16,128,1,fp8,fp8,0,3.115754763285319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,16,8,128,1,float16,fp8,0,9.500426610310873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,1,128,1,float16,fp8,0,4.537301381429036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,1,128,1,fp8,fp8,0,3.0330400466918945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,2,128,1,float16,float16,0,4.512186686197917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,2,128,1,fp8,fp8,0,3.065450668334961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,2,128,1,float16,fp8,0,4.457503954569499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,4,128,1,float16,float16,0,4.502485275268555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,4,128,1,float16,fp8,0,4.662970542907715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,4,128,1,fp8,fp8,0,3.0555413564046225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,8,128,1,float16,float16,0,4.766528129577637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,1,128,1,float16,float16,0,2.3440319697062173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,1,128,1,float16,fp8,0,2.3025546073913574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,8,128,1,fp8,fp8,0,3.1333068211873374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,16,128,1,float16,float16,0,2.280240058898926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,16,8,128,1,float16,fp8,0,4.8142134348551435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,16,128,1,float16,fp8,0,2.2809599240620932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,16,128,1,fp8,fp8,0,1.5799946784973145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,1,128,1,fp8,fp8,0,1.5984320640563965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,2,128,1,float16,float16,0,2.3517173131306968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,2,128,1,float16,fp8,0,2.3063732782999673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,2,128,1,fp8,fp8,0,1.5955840746561687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,4,128,1,float16,float16,0,2.330501397450765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,4,128,1,fp8,fp8,0,1.6045066515604656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,4,128,1,float16,fp8,0,2.3221813837687173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,8,128,1,float16,fp8,0,2.371786594390869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,8,128,1,float16,float16,0,2.3462026913960776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,16,128,1,float16,fp8,0,1.2321279843648274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,1,128,1,float16,float16,0,1.2600800196329753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,16,8,128,1,fp8,fp8,0,1.6412746111551921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,16,128,1,float16,float16,0,1.2326613267262776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,1,128,1,float16,fp8,0,1.249029318491618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,1,128,1,fp8,fp8,0,0.8247679869333903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,16,128,1,fp8,fp8,0,0.822650671005249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,2,128,1,float16,float16,0,1.2622186342875164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,2,128,1,fp8,fp8,0,0.8292586803436279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,2,128,1,float16,fp8,0,1.2531999746958415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,4,128,1,float16,float16,0,1.2742400169372559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,4,128,1,float16,fp8,0,1.2521226406097412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,4,128,1,fp8,fp8,0,0.8337866465250651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,8,128,1,float16,float16,0,1.2842986583709717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,8,128,1,float16,fp8,0,1.2561866442362468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,16,8,128,1,fp8,fp8,0,0.8521599769592285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,1,128,1,float16,fp8,0,6.526442845662435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,1,128,1,float16,float16,0,6.706085205078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,1,128,1,fp8,fp8,0,4.445050557454427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,2,128,1,fp8,fp8,0,4.427344004313151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,2,128,1,float16,float16,0,6.727296193440755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,2,128,1,float16,fp8,0,6.612314860026042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,4,128,1,float16,float16,0,6.581994374593099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,4,128,1,float16,fp8,0,6.8001759847005205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,4,128,1,fp8,fp8,0,4.330848058064778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,1,128,1,float16,float16,0,3.204922676086426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,8,128,1,float16,float16,0,6.706607818603516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,16,128,1,float16,float16,0,3.2125813166300454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,8,128,1,float16,fp8,0,6.698752085367839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,16,128,1,float16,fp8,0,3.1911840438842773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,1,128,1,float16,fp8,0,3.18941338857015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,16,8,128,1,fp8,fp8,0,4.522053400675456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,1,128,1,fp8,fp8,0,2.1932160059611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,16,128,1,fp8,fp8,0,2.2038346926371255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,2,128,1,fp8,fp8,0,2.2441867192586265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,2,128,1,float16,float16,0,3.3198026021321616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,2,128,1,float16,fp8,0,3.1754185358683267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,4,128,1,float16,fp8,0,3.3471946716308594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,4,128,1,fp8,fp8,0,2.227952003479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,4,128,1,float16,float16,0,3.2329867680867515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,8,128,1,float16,float16,0,3.3714027404785156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,1,128,1,float16,float16,0,1.702565352121989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,16,128,1,float16,float16,0,1.6613012949625652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,1,128,1,fp8,fp8,0,1.1622773011525471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,16,128,1,fp8,fp8,0,1.1655840078989665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,8,128,1,float16,fp8,0,3.2072693506876626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,1,128,1,float16,fp8,0,1.6644479433695476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,16,8,128,1,fp8,fp8,0,2.3060320218404136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,16,128,1,float16,fp8,0,1.6456054051717122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,2,128,1,float16,float16,0,1.7076266606648762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,2,128,1,fp8,fp8,0,1.1651466687520344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,2,128,1,float16,fp8,0,1.6661334037780762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,4,128,1,float16,float16,0,1.6901599566141765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,4,128,1,float16,fp8,0,1.6674985885620117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,8,128,1,float16,float16,0,1.7337120374043782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,4,128,1,fp8,fp8,0,1.1737173398335774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,8,128,1,fp8,fp8,0,1.1888586680094402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,16,8,128,1,float16,fp8,0,1.6778133710225422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,1,128,1,float16,float16,0,0.9239412943522135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,16,128,1,float16,float16,0,0.905290683110555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,16,128,1,float16,fp8,0,0.900170644124349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,16,128,1,fp8,fp8,0,0.6102826595306396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,1,128,1,float16,fp8,0,0.9160213470458984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,2,128,1,float16,float16,0,0.9323360125223795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,1,128,1,fp8,fp8,0,0.6084479888280233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,2,128,1,float16,fp8,0,0.9148853619893392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,2,128,1,fp8,fp8,0,0.6122293472290039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,4,128,1,float16,float16,0,0.9261439641316732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,4,128,1,float16,fp8,0,0.917248010635376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,4,128,1,fp8,fp8,0,0.6140106519063314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,8,128,1,float16,float16,0,0.9406720002492269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,8,128,1,float16,fp8,0,0.9311520258585612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,16,8,128,1,fp8,fp8,0,0.624789317448934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,1,128,1,float16,fp8,0,8.555861155192057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,1,128,1,fp8,fp8,0,5.708351771036784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,1,128,1,float16,float16,0,8.826698939005533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,2,128,1,float16,fp8,0,8.866704305013021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,2,128,1,fp8,fp8,0,5.973946889241536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,2,128,1,float16,float16,0,8.948981602986654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,4,128,1,float16,float16,0,8.986005147298178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,4,128,1,float16,fp8,0,8.921781539916992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,1,128,1,float16,float16,0,4.296496073404948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,4,128,1,fp8,fp8,0,5.760789235432942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,16,128,1,float16,fp8,0,4.159568150838216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,16,128,1,fp8,fp8,0,2.9638773600260415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,8,128,1,float16,float16,0,9.023040135701498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,16,128,1,float16,float16,0,4.4986880620320635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,1,128,1,float16,fp8,0,4.154954592386882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,1,128,1,fp8,fp8,0,2.8886613845825195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,8,128,1,float16,fp8,0,8.870357513427734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,16,8,128,1,fp8,fp8,0,5.927504221598308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,2,128,1,float16,float16,0,4.202170689900716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,2,128,1,float16,fp8,0,4.2718079884847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,2,128,1,fp8,fp8,0,2.8952159881591797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,4,128,1,float16,fp8,0,4.361034711201985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,4,128,1,fp8,fp8,0,2.915424029032389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,4,128,1,float16,float16,0,4.436506589253743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,8,128,1,float16,fp8,0,4.43122132619222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,1,128,1,float16,float16,0,2.1384480794270835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,16,128,1,float16,float16,0,2.18340269724528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,8,128,1,float16,float16,0,4.392298698425293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,1,128,1,float16,fp8,0,2.143376032511393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,16,128,1,float16,fp8,0,2.1241866747538247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,16,8,128,1,fp8,fp8,0,2.9615891774495444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,1,128,1,fp8,fp8,0,1.4914453824361165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,2,128,1,float16,fp8,0,2.1608853340148926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,16,128,1,fp8,fp8,0,1.519968032836914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,2,128,1,float16,float16,0,2.1438345909118652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,4,128,1,float16,float16,0,2.2074400583902993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,2,128,1,fp8,fp8,0,1.4974613189697266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,4,128,1,float16,fp8,0,2.132767995198568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,4,128,1,fp8,fp8,0,1.5139519373575847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,16,128,1,float16,float16,0,1.1228586832682292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,8,128,1,float16,float16,0,2.1843199729919434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,1,128,1,float16,float16,0,1.135434627532959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,8,128,1,float16,fp8,0,2.192464033762614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,1,128,1,float16,fp8,0,1.1222346623738606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,16,128,1,float16,fp8,0,1.16156800587972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,16,128,1,fp8,fp8,0,0.8101173241933187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,16,8,128,1,fp8,fp8,0,1.5418613751729329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,1,128,1,fp8,fp8,0,0.8027146657307943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,2,128,1,float16,fp8,0,1.1241652965545654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,2,128,1,float16,float16,0,1.140506664911906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,2,128,1,fp8,fp8,0,0.8037386735280355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,4,128,1,float16,float16,0,1.1434293588002522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,4,128,1,float16,fp8,0,1.1457493305206299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,4,128,1,fp8,fp8,0,0.8095893065134684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,1,128,1,float16,float16,0,0.6432480017344157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,16,128,1,float16,float16,0,0.628383994102478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,8,128,1,float16,fp8,0,1.1366933186848958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,16,128,1,float16,fp8,0,0.6286240021387736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,8,128,1,fp8,fp8,0,0.8201759656270345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,16,8,128,1,float16,float16,0,1.152453343073527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,16,128,1,fp8,fp8,0,0.42847466468811035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,1,128,1,fp8,fp8,0,0.429418683052063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,1,128,1,float16,fp8,0,0.6306986808776855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,2,128,1,float16,fp8,0,0.6297866503397623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,2,128,1,float16,float16,0,0.6399733225504557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,2,128,1,fp8,fp8,0,0.4238986571629842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,4,128,1,float16,float16,0,0.6412373383839926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,4,128,1,fp8,fp8,0,0.42692800362904865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,4,128,1,float16,fp8,0,0.6356906493504842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,8,128,1,float16,float16,0,0.6537493467330933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,8,128,1,fp8,fp8,0,0.4334506591161092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,16,8,128,1,float16,fp8,0,0.6370613177617391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,1,128,1,fp8,fp8,0,3.456933339436849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,1,128,1,float16,fp8,0,5.260586738586426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,1,128,1,float16,float16,0,5.196373303731282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,2,128,1,float16,float16,0,5.201007843017578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,2,128,1,fp8,fp8,0,3.6422398885091147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,2,128,1,float16,fp8,0,5.168592135111491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,4,128,1,float16,float16,0,5.175168037414551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,4,128,1,float16,fp8,0,4.946736017862956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,4,128,1,fp8,fp8,0,3.5079145431518555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,1,128,1,float16,float16,0,2.503002643585205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,16,128,1,float16,float16,0,2.749717394510905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,8,128,1,float16,float16,0,5.308431943257649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,16,128,1,float16,fp8,0,2.520688056945801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,8,128,1,float16,fp8,0,5.056495984395345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,16,128,1,fp8,fp8,0,1.8170933723449707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,16,8,128,1,fp8,fp8,0,3.584122657775879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,1,128,1,float16,fp8,0,2.4690346717834473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,1,128,1,fp8,fp8,0,1.7698079744974773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,2,128,1,float16,float16,0,2.5988160769144693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,2,128,1,float16,fp8,0,2.5203253428141275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,2,128,1,fp8,fp8,0,1.7833493550618489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,4,128,1,float16,float16,0,2.5371200243631997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,4,128,1,float16,fp8,0,2.4876747131347656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,4,128,1,fp8,fp8,0,1.7940533955891926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,16,128,1,float16,float16,0,1.3301866849263508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,8,128,1,float16,float16,0,2.5553760528564453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,1,128,1,float16,float16,0,1.2942933241526287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,8,128,1,fp8,fp8,0,1.9296372731526692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,1,128,1,float16,fp8,0,1.2779093583424885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,16,8,128,1,float16,fp8,0,2.5189387003580728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,16,128,1,float16,fp8,0,1.3000319798787434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,16,128,1,fp8,fp8,0,0.9555892944335938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,1,128,1,fp8,fp8,0,0.923695961634318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,2,128,1,float16,fp8,0,1.2809600035349529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,2,128,1,fp8,fp8,0,0.947381337483724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,2,128,1,float16,float16,0,1.2991573015848796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,4,128,1,float16,float16,0,1.3068586985270183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,4,128,1,fp8,fp8,0,0.9376853307088217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,4,128,1,float16,fp8,0,1.2875413099924724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,16,128,1,float16,float16,0,0.7400853633880615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,16,128,1,float16,fp8,0,0.6923627058664957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,8,128,1,float16,float16,0,1.324405352274577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,8,128,1,float16,fp8,0,1.3226346969604492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,16,8,128,1,fp8,fp8,0,0.9570826689402262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,1,128,1,float16,float16,0,0.7013599872589111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,1,128,1,float16,fp8,0,0.6913173198699951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,16,128,1,fp8,fp8,0,0.5046453475952148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,2,128,1,float16,fp8,0,0.6935253143310547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,2,128,1,float16,float16,0,0.7023306687672933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,1,128,1,fp8,fp8,0,0.49118932088216144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,2,128,1,fp8,fp8,0,0.482965350151062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,4,128,1,float16,float16,0,0.7040639718373617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,4,128,1,fp8,fp8,0,0.48792000611623126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,4,128,1,float16,fp8,0,0.6987946828206381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,8,128,1,float16,float16,0,0.7099146842956543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,16,128,1,float16,fp8,0,0.35658665498097736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,8,128,1,float16,fp8,0,0.7026507059733073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,16,128,1,fp8,fp8,0,0.2781386574109395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,16,8,128,1,fp8,fp8,0,0.502784013748169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,16,128,1,float16,float16,0,0.37057065963745117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,1,128,1,float16,float16,0,0.3543039957682292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,1,128,1,float16,fp8,0,0.3504800001780192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,1,128,1,fp8,fp8,0,0.27187732855478924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,2,128,1,float16,float16,0,0.3549333413441976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,2,128,1,float16,fp8,0,0.3490826686223348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,2,128,1,fp8,fp8,0,0.27245332797368366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,4,128,1,float16,float16,0,0.35753599802652997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,4,128,1,float16,fp8,0,0.35442133744557697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,4,128,1,fp8,fp8,0,0.27513599395751953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,8,128,1,float16,fp8,0,0.35577599207560223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,8,128,1,float16,float16,0,0.3612159887949626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,16,8,128,1,fp8,fp8,0,0.27774399518966675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,1,128,1,float16,float16,0,5.0335038503011065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,1,128,1,float16,fp8,0,4.770069440205892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,1,128,1,fp8,fp8,0,3.5185012817382812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,2,128,1,float16,float16,0,5.394533157348633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,2,128,1,float16,fp8,0,5.0104319254557295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,2,128,1,fp8,fp8,0,3.5524587631225586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,4,128,1,float16,fp8,0,5.138053258260091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,4,128,1,float16,float16,0,5.070576032002767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,1,128,1,float16,float16,0,2.4590880076090493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,4,128,1,fp8,fp8,0,3.5852746963500977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,16,128,1,float16,float16,0,2.5329972902933755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,16,128,1,float16,fp8,0,2.4981813430786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,1,128,1,float16,fp8,0,2.4147466023763022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,16,128,1,fp8,fp8,0,1.8955413500467937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,8,128,1,float16,float16,0,5.140149434407552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,8,128,1,float16,fp8,0,4.951829274495442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,1,128,1,fp8,fp8,0,1.7849653561909993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,16,8,128,1,fp8,fp8,0,4.007823944091797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,2,128,1,float16,float16,0,2.4613067309061685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,2,128,1,float16,fp8,0,2.419989267985026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,2,128,1,fp8,fp8,0,1.8016212781270344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,4,128,1,float16,float16,0,2.5243040720621743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,4,128,1,float16,fp8,0,2.434607982635498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,4,128,1,fp8,fp8,0,1.819658597310384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,8,128,1,float16,float16,0,2.5339892705281577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,8,128,1,float16,fp8,0,2.6880321502685547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,1,128,1,float16,float16,0,1.248026688893636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,1,128,1,float16,fp8,0,1.2268586953481038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,16,128,1,float16,float16,0,1.2948426405588787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,16,128,1,float16,fp8,0,1.277781327565511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,1,128,1,fp8,fp8,0,0.914410670598348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,16,128,1,fp8,fp8,0,0.974671999613444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,16,8,128,1,fp8,fp8,0,1.8679307301839192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,2,128,1,float16,float16,0,1.296229362487793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,2,128,1,float16,fp8,0,1.228501319885254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,2,128,1,fp8,fp8,0,0.9228426615397135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,4,128,1,float16,float16,0,1.264405330022176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,4,128,1,fp8,fp8,0,0.9348533153533936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,4,128,1,float16,fp8,0,1.244282643000285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,16,128,1,float16,float16,0,0.6721226374308268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,8,128,1,float16,float16,0,1.2990187009175618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,16,128,1,float16,fp8,0,0.6644159952799479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,8,128,1,fp8,fp8,0,1.010464032491048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,1,128,1,float16,float16,0,0.6616160074869791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,1,128,1,float16,fp8,0,0.6502879858016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,16,8,128,1,float16,fp8,0,1.2597546577453613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,16,128,1,fp8,fp8,0,0.519157330195109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,2,128,1,float16,fp8,0,0.6515040000279745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,2,128,1,float16,float16,0,0.6681493123372396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,1,128,1,fp8,fp8,0,0.4904799858729045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,2,128,1,fp8,fp8,0,0.5073546568552653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,4,128,1,float16,float16,0,0.668293317159017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,4,128,1,float16,fp8,0,0.6577599843343099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,4,128,1,fp8,fp8,0,0.49956266085306805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,8,128,1,float16,float16,0,0.6764266490936279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,8,128,1,float16,fp8,0,0.6652533213297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,16,128,1,float16,fp8,0,0.3887999852498372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,16,8,128,1,fp8,fp8,0,0.5115413268407186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,16,128,1,fp8,fp8,0,0.27713600794474286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,1,128,1,float16,float16,0,0.36976532141367596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,16,128,1,float16,float16,0,0.3720906575520833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,1,128,1,fp8,fp8,0,0.26043200492858887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,2,128,1,float16,float16,0,0.37062398592631024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,2,128,1,float16,fp8,0,0.3723093271255493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,2,128,1,fp8,fp8,0,0.260261336962382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,1,128,1,float16,fp8,0,0.36254934469858807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,4,128,1,float16,float16,0,0.3739466667175293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,4,128,1,fp8,fp8,0,0.26494399706522626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,4,128,1,float16,fp8,0,0.3678986628850301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,16,128,1,float16,float16,0,0.19355199734369913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,8,128,1,float16,fp8,0,0.37174399693806964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,8,128,1,float16,float16,0,0.377125342686971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,16,8,128,1,fp8,fp8,0,0.2807573278745015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,16,128,1,fp8,fp8,0,0.15794666608174643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,1,128,1,float16,float16,0,0.1869706710179647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,16,128,1,float16,fp8,0,0.18976000944773355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,1,128,1,fp8,fp8,0,0.1492586632569631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,2,128,1,float16,float16,0,0.18927466869354248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,2,128,1,float16,fp8,0,0.18573333819707236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,1,128,1,float16,fp8,0,0.18538665771484375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,4,128,1,float16,float16,0,0.19031467040379843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,2,128,1,fp8,fp8,0,0.153519997994105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,4,128,1,float16,fp8,0,0.18615466356277466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,8,128,1,float16,float16,0,0.19248533248901367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,8,128,1,float16,fp8,0,0.1891040007273356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,4,128,1,fp8,fp8,0,0.15427199999491373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,16,8,128,1,fp8,fp8,0,0.1570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,1,128,1,float16,fp8,0,2.9607626597086587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,1,128,1,fp8,fp8,0,2.428149382273356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,1,128,1,float16,float16,0,3.024832089742025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,2,128,1,float16,fp8,0,3.0161708196004233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,2,128,1,fp8,fp8,0,2.2651999791463218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,2,128,1,float16,float16,0,3.03659725189209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,4,128,1,float16,float16,0,3.0752960840861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,4,128,1,float16,fp8,0,3.031701405843099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,1,128,1,float16,float16,0,1.5204319953918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,4,128,1,fp8,fp8,0,2.297536055246989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,16,128,1,float16,float16,0,1.6022772789001465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,16,128,1,float16,fp8,0,1.560922622680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,8,128,1,float16,float16,0,3.4993600845336914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,1,128,1,float16,fp8,0,1.4870506922403972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,8,128,1,fp8,fp8,0,2.3701119422912598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,16,128,1,fp8,fp8,0,1.2298826376597087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,1,128,1,fp8,fp8,0,1.1461546421051025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,16,8,128,1,float16,fp8,0,3.074485460917155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,2,128,1,float16,float16,0,1.5265919367472331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,2,128,1,float16,fp8,0,1.4930933316548665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,2,128,1,fp8,fp8,0,1.2217439810434978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,4,128,1,float16,float16,0,1.5420692761739094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,4,128,1,float16,fp8,0,1.5262773831685383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,4,128,1,fp8,fp8,0,1.1697546641031902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,8,128,1,float16,float16,0,1.5770400365193684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,1,128,1,float16,float16,0,0.7892320156097412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,8,128,1,float16,fp8,0,1.5433653195699055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,16,128,1,float16,float16,0,0.8192586898803711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,16,128,1,float16,fp8,0,0.8934079806009928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,16,8,128,1,fp8,fp8,0,1.206485350926717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,1,128,1,float16,fp8,0,0.7665812969207764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,16,128,1,fp8,fp8,0,0.6397279898325602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,1,128,1,fp8,fp8,0,0.5911306540171305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,2,128,1,float16,float16,0,0.7927467028299967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,2,128,1,float16,fp8,0,0.7717226346333822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,4,128,1,float16,float16,0,0.8350613117218018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,2,128,1,fp8,fp8,0,0.5966720183690389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,4,128,1,float16,fp8,0,0.7783040205637614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,4,128,1,fp8,fp8,0,0.606223980585734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,8,128,1,float16,float16,0,0.8082239627838135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,8,128,1,float16,fp8,0,0.7941919962565104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,1,128,1,float16,float16,0,0.422762672106425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,16,8,128,1,fp8,fp8,0,0.6235520044962565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,16,128,1,float16,float16,0,0.43693331877390545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,16,128,1,float16,fp8,0,0.47162667910257977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,1,128,1,float16,fp8,0,0.4142666657765706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,1,128,1,fp8,fp8,0,0.309989333152771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,2,128,1,float16,float16,0,0.424453337987264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,16,128,1,fp8,fp8,0,0.34759998321533203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,2,128,1,float16,fp8,0,0.4160319964090983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,2,128,1,fp8,fp8,0,0.31279999017715454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,4,128,1,float16,fp8,0,0.43504532178243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,4,128,1,float16,float16,0,0.4297279914220174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,4,128,1,fp8,fp8,0,0.31747732559839886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,8,128,1,float16,float16,0,0.4344000021616618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,8,128,1,float16,fp8,0,0.4265120029449463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,16,128,1,float16,float16,0,0.2273599902788798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,16,128,1,float16,fp8,0,0.224346657594045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,16,8,128,1,fp8,fp8,0,0.3322880069414775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,1,128,1,float16,float16,0,0.2169119914372762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,16,128,1,fp8,fp8,0,0.19909866650899252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,1,128,1,float16,fp8,0,0.21234667301177979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,2,128,1,float16,float16,0,0.21773332357406616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,2,128,1,float16,fp8,0,0.21289600928624472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,1,128,1,fp8,fp8,0,0.17244799931844076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,4,128,1,float16,float16,0,0.21877866983413696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,2,128,1,fp8,fp8,0,0.1732106606165568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,4,128,1,fp8,fp8,0,0.1823520064353943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,4,128,1,float16,fp8,0,0.2149333357810974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,16,128,1,float16,float16,0,0.13219733039538065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,8,128,1,fp8,fp8,0,0.18124266465504965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,8,128,1,float16,float16,0,0.2249013384183248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,1,128,1,float16,float16,0,0.12691199779510498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,16,128,1,float16,fp8,0,0.12934399644533792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,16,8,128,1,float16,fp8,0,0.22078933318456015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,1,128,1,float16,fp8,0,0.12517333030700684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,16,128,1,fp8,fp8,0,0.11202667156855266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,1,128,1,fp8,fp8,0,0.1013813316822052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,2,128,1,float16,float16,0,0.12757333119710287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,2,128,1,float16,fp8,0,0.12507200241088867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,2,128,1,fp8,fp8,0,0.10281067093213399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,4,128,1,float16,float16,0,0.12876799702644348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,4,128,1,float16,fp8,0,0.12678933143615723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,8,128,1,float16,float16,0,0.13103999694188437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,4,128,1,fp8,fp8,0,0.10525866349538167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,8,128,1,fp8,fp8,0,0.10965866843859355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,16,8,128,1,float16,fp8,0,0.12890133261680603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,1,128,1,float16,fp8,0,3.4091574350992837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,2,128,1,float16,float16,0,3.25600528717041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,1,128,1,float16,float16,0,3.2350133260091147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,2,128,1,float16,fp8,0,3.1524585088094077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,1,128,1,fp8,fp8,0,2.5030666987101235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,2,128,1,fp8,fp8,0,2.5242613156636557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,4,128,1,float16,float16,0,3.294933319091797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,4,128,1,float16,fp8,0,3.1533225377400718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,1,128,1,float16,float16,0,1.6075785954793294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,4,128,1,fp8,fp8,0,2.8395360310872397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,16,128,1,float16,fp8,0,1.673221270243327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,1,128,1,float16,fp8,0,1.5469813346862793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,16,128,1,float16,float16,0,1.7237599690755208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,16,128,1,fp8,fp8,0,1.3754293123881023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,8,128,1,float16,float16,0,3.4126879374186196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,1,128,1,fp8,fp8,0,1.3570666313171387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,8,128,1,fp8,fp8,0,2.6438186963399253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,2,128,1,float16,float16,0,1.6202452977498372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,16,8,128,1,float16,fp8,0,3.233306566874186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,2,128,1,fp8,fp8,0,1.2589279810587566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,2,128,1,float16,fp8,0,1.5543786684672039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,4,128,1,float16,float16,0,1.6380000114440918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,4,128,1,fp8,fp8,0,1.2807093461354573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,4,128,1,float16,fp8,0,1.7530986467997234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,16,128,1,float16,float16,0,0.868880033493042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,1,128,1,float16,float16,0,0.8090613683064779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,8,128,1,float16,float16,0,1.703290621439616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,16,128,1,float16,fp8,0,0.849290688832601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,1,128,1,float16,fp8,0,0.8196907043457031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,8,128,1,float16,fp8,0,1.6217333475748699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,16,128,1,fp8,fp8,0,0.7046773433685303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,16,8,128,1,fp8,fp8,0,1.3281973203023274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,1,128,1,fp8,fp8,0,0.6349759896596273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,2,128,1,float16,float16,0,0.8091733455657959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,2,128,1,fp8,fp8,0,0.6413173278172811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,2,128,1,float16,fp8,0,0.7873706817626953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,4,128,1,float16,float16,0,0.8816373348236084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,4,128,1,fp8,fp8,0,0.6535626649856567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,4,128,1,float16,fp8,0,0.8012373447418213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,16,128,1,float16,fp8,0,0.44440531730651855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,16,128,1,float16,float16,0,0.457477331161499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,8,128,1,float16,fp8,0,0.8163893222808838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,8,128,1,fp8,fp8,0,0.6776159604390463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,1,128,1,float16,float16,0,0.42419199148813885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,16,128,1,fp8,fp8,0,0.415669322013855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,1,128,1,float16,fp8,0,0.4137760003407796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,16,8,128,1,float16,float16,0,0.8383626937866211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,1,128,1,fp8,fp8,0,0.3376266558965047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,2,128,1,fp8,fp8,0,0.3412533203760783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,2,128,1,float16,fp8,0,0.41539732615152997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,2,128,1,float16,float16,0,0.4286506573359172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,4,128,1,float16,float16,0,0.4318346579869588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,4,128,1,float16,fp8,0,0.4438399871190389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,4,128,1,fp8,fp8,0,0.3473866780598958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,1,128,1,float16,float16,0,0.23974400758743286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,16,128,1,float16,fp8,0,0.2428320050239563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,8,128,1,float16,float16,0,0.4405866861343384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,16,128,1,float16,float16,0,0.24690133333206177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,8,128,1,fp8,fp8,0,0.3572053511937459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,16,8,128,1,float16,fp8,0,0.4309600194295247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,16,128,1,fp8,fp8,0,0.20185067256291708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,1,128,1,fp8,fp8,0,0.1856266657511393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,1,128,1,float16,fp8,0,0.22873600323994955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,2,128,1,float16,float16,0,0.23653332392374674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,2,128,1,float16,fp8,0,0.2307200034459432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,4,128,1,float16,float16,0,0.23938665787378946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,2,128,1,fp8,fp8,0,0.1803893248240153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,4,128,1,float16,fp8,0,0.23357866207758585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,4,128,1,fp8,fp8,0,0.18270933628082275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,8,128,1,float16,float16,0,0.2610879937807719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,1,128,1,float16,float16,0,0.12099200487136841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,8,128,1,fp8,fp8,0,0.19092265764872232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,1,128,1,float16,fp8,0,0.11909332871437073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,16,128,1,float16,fp8,0,0.12807466586430868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,16,128,1,float16,float16,0,0.13005333145459494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,16,128,1,fp8,fp8,0,0.11370666821797688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,16,8,128,1,float16,fp8,0,0.2393760085105896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,1,128,1,fp8,fp8,0,0.1032319962978363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,2,128,1,float16,float16,0,0.12195199728012085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,2,128,1,float16,fp8,0,0.11925333738327026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,4,128,1,float16,float16,0,0.12346133589744568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,2,128,1,fp8,fp8,0,0.10315733154614766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,4,128,1,float16,fp8,0,0.1209279994169871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,4,128,1,fp8,fp8,0,0.10586133599281311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,8,128,1,float16,float16,0,0.12718400359153748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,8,128,1,float16,fp8,0,0.1285546620686849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,16,128,1,float16,float16,0,0.07547200222810109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,1,128,1,float16,float16,0,0.07247466842333476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,16,8,128,1,fp8,fp8,0,0.10912000139554341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,1,128,1,float16,fp8,0,0.07145600020885468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,16,128,1,float16,fp8,0,0.0751039981842041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,16,128,1,fp8,fp8,0,0.0701386680205663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,2,128,1,float16,float16,0,0.07263466715812683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,2,128,1,float16,fp8,0,0.07140799860159557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,1,128,1,fp8,fp8,0,0.06239999830722809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,2,128,1,fp8,fp8,0,0.06221333146095276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,4,128,1,float16,float16,0,0.07303466896216075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,4,128,1,float16,fp8,0,0.07178666690985362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,8,128,1,float16,float16,0,0.07412266731262207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,8,128,1,float16,fp8,0,0.07281599938869476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,4,128,1,fp8,fp8,0,0.06351999938488007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,16,8,128,1,fp8,fp8,0,0.06564266482988994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,1,128,1,float16,float16,0,2.1192320187886557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,1,128,1,fp8,fp8,0,1.707738717397054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,1,128,1,float16,fp8,0,1.990623950958252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,2,128,1,float16,float16,0,2.1266132990519204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,2,128,1,float16,fp8,0,2.0003786087036133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,2,128,1,fp8,fp8,0,1.8921279907226562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,4,128,1,float16,fp8,0,2.041930675506592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,4,128,1,float16,float16,0,2.164170742034912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,4,128,1,fp8,fp8,0,1.7458133697509766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,16,128,1,float16,float16,0,1.1618133385976155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,16,128,1,float16,fp8,0,1.2949972947438557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,1,128,1,float16,float16,0,1.0472959677378337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,16,128,1,fp8,fp8,0,0.9535946846008301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,8,128,1,fp8,fp8,0,1.7944213549296062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,8,128,1,float16,float16,0,2.2485547065734863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,16,8,128,1,float16,fp8,0,2.0810240109761557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,1,128,1,fp8,fp8,0,0.8514666557312012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,2,128,1,float16,float16,0,1.1638933022816975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,2,128,1,float16,fp8,0,1.0037120183308919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,1,128,1,float16,fp8,0,1.0017333030700684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,2,128,1,fp8,fp8,0,0.8591573238372803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,4,128,1,float16,float16,0,1.071893294652303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,4,128,1,float16,fp8,0,1.0178186893463135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,4,128,1,fp8,fp8,0,0.8788426717122396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,16,128,1,float16,fp8,0,0.5702559947967529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,1,128,1,float16,float16,0,0.5315893491109213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,16,128,1,float16,float16,0,0.5863893429438273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,8,128,1,fp8,fp8,0,0.9063200155893961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,8,128,1,float16,fp8,0,1.1820586522420247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,16,8,128,1,float16,float16,0,1.1114719708760579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,16,128,1,fp8,fp8,0,0.4919733206431071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,1,128,1,fp8,fp8,0,0.4352213144302368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,1,128,1,float16,fp8,0,0.5144746700922648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,2,128,1,float16,float16,0,0.5340640147527059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,2,128,1,fp8,fp8,0,0.44117867946624756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,2,128,1,float16,fp8,0,0.5188159942626953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,4,128,1,float16,float16,0,0.5864799817403158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,4,128,1,float16,fp8,0,0.5244853496551514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,4,128,1,fp8,fp8,0,0.44795199235280353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,8,128,1,float16,float16,0,0.5572746594746908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,16,128,1,float16,float16,0,0.30964799722035724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,1,128,1,float16,float16,0,0.2842560013135274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,16,128,1,fp8,fp8,0,0.3589706818262736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,8,128,1,fp8,fp8,0,0.4660586516062419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,16,128,1,float16,fp8,0,0.3015999992688497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,1,128,1,float16,fp8,0,0.27516265710194904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,16,8,128,1,float16,fp8,0,0.5383466482162476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,1,128,1,fp8,fp8,0,0.2254506746927897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,2,128,1,float16,float16,0,0.2853920062383016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,2,128,1,float16,fp8,0,0.27822933594385785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,2,128,1,fp8,fp8,0,0.24092799425125122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,4,128,1,float16,float16,0,0.2898400028546651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,4,128,1,fp8,fp8,0,0.23228265841801962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,4,128,1,float16,fp8,0,0.28200000524520874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,8,128,1,float16,fp8,0,0.28838932514190674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,16,128,1,float16,float16,0,0.16941332817077637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,16,128,1,float16,fp8,0,0.21142399311065674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,8,128,1,fp8,fp8,0,0.24574933449427286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,16,8,128,1,float16,float16,0,0.2972693244616191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,16,128,1,fp8,fp8,0,0.14135467012723288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,1,128,1,float16,float16,0,0.14949867129325867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,1,128,1,fp8,fp8,0,0.12557333707809448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,2,128,1,float16,fp8,0,0.15135467052459717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,2,128,1,float16,float16,0,0.15074132879575095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,1,128,1,float16,fp8,0,0.1460533340771993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,2,128,1,fp8,fp8,0,0.12611200412114462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,4,128,1,float16,fp8,0,0.14858133594195047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,4,128,1,float16,float16,0,0.15251200397809347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,4,128,1,fp8,fp8,0,0.12940266728401184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,16,128,1,float16,fp8,0,0.08992532889048259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,8,128,1,float16,fp8,0,0.16658666729927063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,8,128,1,fp8,fp8,0,0.13521599769592285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,16,128,1,float16,float16,0,0.0916426678498586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,16,8,128,1,float16,float16,0,0.15864533185958862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,16,128,1,fp8,fp8,0,0.08237333099047343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,1,128,1,float16,fp8,0,0.08137600123882294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,1,128,1,float16,float16,0,0.0830026666323344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,1,128,1,fp8,fp8,0,0.07131200035413106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,2,128,1,float16,float16,0,0.08366400003433228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,2,128,1,float16,fp8,0,0.08276799817879994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,4,128,1,float16,float16,0,0.08629866441090901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,4,128,1,float16,fp8,0,0.0846506655216217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,4,128,1,fp8,fp8,0,0.07657066484292348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,2,128,1,fp8,fp8,0,0.07262399792671204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,8,128,1,float16,fp8,0,0.08658666412035625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,16,128,1,float16,float16,0,0.054799998799959816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,16,128,1,float16,fp8,0,0.05469333132108053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,8,128,1,float16,float16,0,0.08814400434494019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,1,128,1,float16,float16,0,0.05366933345794678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,16,128,1,fp8,fp8,0,0.05287999908129374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,16,8,128,1,fp8,fp8,0,0.08385599652926128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,1,128,1,float16,fp8,0,0.05310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,1,128,1,fp8,fp8,0,0.04699199895064036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,2,128,1,float16,float16,0,0.053861334919929504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,2,128,1,float16,fp8,0,0.05291733145713806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,4,128,1,float16,float16,0,0.05381333331267039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,4,128,1,float16,fp8,0,0.05330666899681091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,2,128,1,fp8,fp8,0,0.04711999992529551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,4,128,1,fp8,fp8,0,0.04774933556715647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,8,128,1,float16,fp8,0,0.05379199981689453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,8,128,1,fp8,fp8,0,0.04941866795221964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,16,8,128,1,float16,float16,0,0.05461333195368449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,1,128,1,float16,float16,0,1.9190559387207031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,1,128,1,float16,fp8,0,1.9078613917032878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,1,128,1,fp8,fp8,0,1.6475520133972168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,2,128,1,fp8,fp8,0,1.7416906356811523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,2,128,1,float16,fp8,0,2.1304640769958496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,2,128,1,float16,float16,0,1.9282933870951335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,4,128,1,float16,fp8,0,2.082533359527588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,4,128,1,float16,float16,0,2.0624160766601562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,1,128,1,float16,float16,0,0.9686826864878336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,4,128,1,fp8,fp8,0,1.8868799209594727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,16,128,1,float16,float16,0,1.6061333020528157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,1,128,1,float16,fp8,0,0.9681440194447836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,8,128,1,float16,float16,0,2.1634186108907065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,16,128,1,float16,fp8,0,1.1958399613698323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,1,128,1,fp8,fp8,0,0.8233866691589355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,16,128,1,fp8,fp8,0,1.0289386908213298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,8,128,1,fp8,fp8,0,1.9092213312784831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,16,8,128,1,float16,fp8,0,2.153066635131836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,2,128,1,float16,float16,0,1.0891893704732258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,2,128,1,float16,fp8,0,0.9768426418304443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,2,128,1,fp8,fp8,0,0.8803359667460123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,4,128,1,float16,fp8,0,1.0308480262756348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,4,128,1,fp8,fp8,0,0.9545173645019531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,4,128,1,float16,float16,0,1.0107839902242024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,8,128,1,float16,float16,0,1.086037317911784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,8,128,1,float16,fp8,0,1.069375991821289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,1,128,1,float16,fp8,0,0.49850134054819745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,1,128,1,float16,float16,0,0.49890132745107013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,1,128,1,fp8,fp8,0,0.41807464758555096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,16,128,1,float16,fp8,0,0.5985813140869141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,16,128,1,float16,float16,0,0.6252906719843546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,16,8,128,1,fp8,fp8,0,1.0642826557159424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,2,128,1,float16,float16,0,0.504095991452535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,16,128,1,fp8,fp8,0,0.5229066610336304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,2,128,1,float16,fp8,0,0.502565344174703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,4,128,1,float16,float16,0,0.5225706497828165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,2,128,1,fp8,fp8,0,0.44889068603515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,4,128,1,float16,fp8,0,0.5148640076319376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,4,128,1,fp8,fp8,0,0.4854453404744466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,8,128,1,float16,float16,0,0.669818639755249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,8,128,1,float16,fp8,0,0.5427093505859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,16,8,128,1,fp8,fp8,0,0.49459731578826904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,1,128,1,float16,fp8,0,0.2620159983634949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,1,128,1,float16,float16,0,0.2620319922765096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,1,128,1,fp8,fp8,0,0.22930665810902914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,16,128,1,float16,fp8,0,0.313482662041982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,2,128,1,float16,float16,0,0.26553599039713544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,16,128,1,float16,float16,0,0.3226026693979899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,2,128,1,float16,fp8,0,0.2646186749140422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,2,128,1,fp8,fp8,0,0.2300106684366862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,4,128,1,float16,float16,0,0.2744373281796773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,16,128,1,fp8,fp8,0,0.27033066749572754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,4,128,1,float16,fp8,0,0.30542399485905963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,4,128,1,fp8,fp8,0,0.25150400400161743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,8,128,1,float16,float16,0,0.2887093424797058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,16,128,1,float16,float16,0,0.17377599080403647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,8,128,1,float16,fp8,0,0.28388800223668414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,16,128,1,float16,fp8,0,0.1685439944267273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,16,8,128,1,fp8,fp8,0,0.2544853289922078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,1,128,1,float16,float16,0,0.15372266372044882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,1,128,1,float16,fp8,0,0.14474667112032572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,1,128,1,fp8,fp8,0,0.11251733700434367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,2,128,1,float16,float16,0,0.1465013325214386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,2,128,1,fp8,fp8,0,0.11449066797892253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,2,128,1,float16,fp8,0,0.14550399780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,16,128,1,fp8,fp8,0,0.14218667149543762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,4,128,1,float16,float16,0,0.15222932895024618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,4,128,1,float16,fp8,0,0.16456533471743265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,4,128,1,fp8,fp8,0,0.1255626678466797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,8,128,1,float16,float16,0,0.1577173372109731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,8,128,1,float16,fp8,0,0.1552853286266327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,16,128,1,float16,float16,0,0.09396800398826599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,1,128,1,float16,float16,0,0.07739200194676717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,16,8,128,1,fp8,fp8,0,0.12895466883977255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,16,128,1,fp8,fp8,0,0.07721066474914551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,1,128,1,float16,fp8,0,0.07652799785137177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,16,128,1,float16,fp8,0,0.09190932909647624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,1,128,1,fp8,fp8,0,0.062319998939832054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,2,128,1,float16,float16,0,0.07770666480064392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,4,128,1,float16,fp8,0,0.07969066500663757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,4,128,1,float16,float16,0,0.08116266628106435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,4,128,1,fp8,fp8,0,0.07012266914049785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,2,128,1,fp8,fp8,0,0.06569600105285645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,2,128,1,float16,fp8,0,0.07726933558781941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,8,128,1,float16,fp8,0,0.0819893330335617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,8,128,1,fp8,fp8,0,0.07057600220044453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,16,8,128,1,float16,float16,0,0.08341333270072937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,16,128,1,float16,float16,0,0.049141332507133484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,1,128,1,float16,float16,0,0.04305600126584371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,16,128,1,float16,fp8,0,0.04833066463470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,1,128,1,float16,fp8,0,0.04322133461634318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,16,128,1,fp8,fp8,0,0.04548799991607666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,1,128,1,fp8,fp8,0,0.036544000109036766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,2,128,1,float16,float16,0,0.043653334180514015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,2,128,1,fp8,fp8,0,0.03765333443880081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,2,128,1,float16,fp8,0,0.04358399907747904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,4,128,1,float16,float16,0,0.04489066700140635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,4,128,1,float16,fp8,0,0.045168002446492515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,8,128,1,float16,float16,0,0.04568533102671305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,4,128,1,fp8,fp8,0,0.039120001097520195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,8,128,1,float16,fp8,0,0.0461706668138504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,16,8,128,1,fp8,fp8,0,0.04168533285458883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,16,128,1,float16,float16,0,0.03311999887228012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,1,128,1,float16,float16,0,0.03179733455181122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,16,128,1,fp8,fp8,0,0.029114666084448498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,1,128,1,float16,fp8,0,0.03169599920511246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,16,128,1,float16,fp8,0,0.03316266586383184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,2,128,1,float16,fp8,0,0.03235200047492981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,2,128,1,float16,float16,0,0.03216533362865448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,4,128,1,float16,float16,0,0.03301866600910822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,2,128,1,fp8,fp8,0,0.0277813325325648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,1,128,1,fp8,fp8,0,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,4,128,1,float16,fp8,0,0.033402666449546814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,4,128,1,fp8,fp8,0,0.028688001135985058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,8,128,1,float16,float16,0,0.03293866664171219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,8,128,1,float16,fp8,0,0.03312533348798752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,16,8,128,1,fp8,fp8,0,0.029189333319664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,1,128,1,float16,fp8,0,1.4920746485392253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,1,128,1,float16,float16,0,1.49946133295695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,2,128,1,float16,fp8,0,1.5070452690124512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,1,128,1,fp8,fp8,0,1.3885653813680012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,2,128,1,float16,float16,0,1.7535306612650554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,2,128,1,fp8,fp8,0,1.4668906529744465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,4,128,1,float16,fp8,0,1.6686933835347493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,4,128,1,float16,float16,0,1.6463200251261394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,1,128,1,float16,float16,0,0.7583839893341064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,4,128,1,fp8,fp8,0,1.6235413551330566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,16,128,1,float16,float16,0,1.511583964029948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,1,128,1,float16,fp8,0,0.7572746276855469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,16,128,1,fp8,fp8,0,0.8947626749674479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,16,128,1,float16,fp8,0,1.0034293333689372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,8,128,1,float16,fp8,0,1.732927958170573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,1,128,1,fp8,fp8,0,0.6848586400349935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,8,128,1,float16,float16,0,1.748949368794759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,16,8,128,1,fp8,fp8,0,1.6577439308166504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,2,128,1,float16,fp8,0,0.7681066989898682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,2,128,1,float16,float16,0,0.8856693108876547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,2,128,1,fp8,fp8,0,0.7373973528544108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,4,128,1,float16,fp8,0,0.8129226366678873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,4,128,1,fp8,fp8,0,0.8168213367462158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,4,128,1,float16,float16,0,0.8032853603363037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,1,128,1,float16,float16,0,0.39135468006134033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,1,128,1,float16,fp8,0,0.3917813301086426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,8,128,1,float16,float16,0,0.8720213572184244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,16,128,1,float16,fp8,0,0.5072106520334879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,16,128,1,float16,float16,0,0.762938658396403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,8,128,1,float16,fp8,0,0.8587573369344076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,16,8,128,1,fp8,fp8,0,0.8321759700775146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,1,128,1,fp8,fp8,0,0.3489760160446167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,16,128,1,fp8,fp8,0,0.4572213490804036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,2,128,1,float16,float16,0,0.3966133197148641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,2,128,1,float16,fp8,0,0.3943466742833455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,2,128,1,fp8,fp8,0,0.3729706605275472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,4,128,1,fp8,fp8,0,0.415333350499471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,4,128,1,float16,fp8,0,0.40646398067474365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,4,128,1,float16,float16,0,0.49218666553497314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,8,128,1,float16,float16,0,0.44596266746520996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,1,128,1,float16,float16,0,0.2055520017941793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,1,128,1,float16,fp8,0,0.23030400276184082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,8,128,1,float16,fp8,0,0.4403093258539836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,16,128,1,float16,float16,0,0.2730506658554077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,16,128,1,float16,fp8,0,0.261898676554362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,16,128,1,fp8,fp8,0,0.23430399099985758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,1,128,1,fp8,fp8,0,0.18347734212875366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,16,8,128,1,fp8,fp8,0,0.42371733983357746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,2,128,1,float16,float16,0,0.20866133769353232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,2,128,1,float16,fp8,0,0.2360960046450297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,2,128,1,fp8,fp8,0,0.19243200620015463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,4,128,1,float16,float16,0,0.21754133701324463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,4,128,1,float16,fp8,0,0.21546665827433267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,4,128,1,fp8,fp8,0,0.2152586579322815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,8,128,1,float16,float16,0,0.23291732867558798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,8,128,1,float16,fp8,0,0.23042132457097372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,1,128,1,float16,float16,0,0.11400533715883891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,1,128,1,float16,fp8,0,0.11348266402880351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,16,8,128,1,fp8,fp8,0,0.2193653384844462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,16,128,1,fp8,fp8,0,0.1258026659488678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,16,128,1,float16,fp8,0,0.14309866229693094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,1,128,1,fp8,fp8,0,0.09546666344006856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,16,128,1,float16,float16,0,0.2044853369394938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,2,128,1,float16,float16,0,0.11637866497039795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,2,128,1,fp8,fp8,0,0.09787733356157939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,2,128,1,float16,fp8,0,0.11518933375676473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,4,128,1,float16,float16,0,0.12140267093976338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,4,128,1,fp8,fp8,0,0.10787733395894368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,4,128,1,float16,fp8,0,0.11994666854540507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,8,128,1,float16,float16,0,0.1264479955037435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,8,128,1,float16,fp8,0,0.15522666772206625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,16,8,128,1,fp8,fp8,0,0.11102933684984843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,16,128,1,float16,float16,0,0.08437866965929668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,16,128,1,float16,fp8,0,0.08124266564846039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,1,128,1,float16,float16,0,0.06379200021425883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,1,128,1,float16,fp8,0,0.06378666559855144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,1,128,1,fp8,fp8,0,0.05401599903901418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,2,128,1,float16,float16,0,0.0661599983771642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,16,128,1,fp8,fp8,0,0.06871466835339864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,2,128,1,fp8,fp8,0,0.056421334544817604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,2,128,1,float16,fp8,0,0.06503466765085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,4,128,1,float16,float16,0,0.06897599995136261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,4,128,1,float16,fp8,0,0.06804800033569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,8,128,1,float16,float16,0,0.06956799825032552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,8,128,1,float16,fp8,0,0.0768746683994929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,16,128,1,float16,float16,0,0.04368533194065094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,4,128,1,fp8,fp8,0,0.0609440008799235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,16,128,1,float16,fp8,0,0.0422986646493276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,16,8,128,1,fp8,fp8,0,0.061797335743904114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,16,128,1,fp8,fp8,0,0.04091199984153112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,1,128,1,float16,float16,0,0.03700799991687139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,1,128,1,float16,fp8,0,0.0369759996732076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,1,128,1,fp8,fp8,0,0.0322026660044988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,2,128,1,float16,float16,0,0.03766933331886927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,2,128,1,float16,fp8,0,0.03724266588687897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,2,128,1,fp8,fp8,0,0.03309333324432373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,4,128,1,fp8,fp8,0,0.035317334036032356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,8,128,1,float16,float16,0,0.03969600051641464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,4,128,1,float16,fp8,0,0.0390079990029335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,8,128,1,float16,fp8,0,0.04089066634575526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,4,128,1,float16,float16,0,0.038912000755469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,16,8,128,1,fp8,fp8,0,0.03698666642109553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,16,128,1,float16,float16,0,0.02712533374627431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,16,128,1,float16,fp8,0,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,1,128,1,float16,float16,0,0.02553066611289978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,1,128,1,fp8,fp8,0,0.023306667804718018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,1,128,1,float16,fp8,0,0.025616000096003216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,16,128,1,fp8,fp8,0,0.02497066557407379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,2,128,1,float16,fp8,0,0.026399999856948853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,2,128,1,float16,float16,0,0.02584533393383026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,2,128,1,fp8,fp8,0,0.02370133250951767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,4,128,1,float16,float16,0,0.02682666728893916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,4,128,1,float16,fp8,0,0.026799999177455902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,4,128,1,fp8,fp8,0,0.02473066747188568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,8,128,1,float16,float16,0,0.02666666607062022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,8,128,1,float16,fp8,0,0.02641066660483678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,16,8,128,1,fp8,fp8,0,0.02492800106604894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,16,128,1,float16,float16,0,0.023210667073726654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,16,128,1,float16,fp8,0,0.023472001155217487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,1,128,1,float16,float16,0,0.022570667167504627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,16,128,1,fp8,fp8,0,0.021125334004561108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,1,128,1,float16,fp8,0,0.022965334355831146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,1,128,1,fp8,fp8,0,0.020506666352351505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,2,128,1,float16,float16,0,0.022469334304332733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,2,128,1,float16,fp8,0,0.022682666778564453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,2,128,1,fp8,fp8,0,0.020560000091791153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,4,128,1,float16,float16,0,0.022863999009132385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,4,128,1,float16,fp8,0,0.02306666721900304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,8,128,1,float16,fp8,0,0.023578666150569916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,8,128,1,float16,float16,0,0.023269332945346832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,4,128,1,fp8,fp8,0,0.021183999876181286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,16,8,128,1,fp8,fp8,0,0.021312000850836437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,1,128,1,float16,fp8,0,0.6540106534957886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,1,128,1,fp8,fp8,0,0.5713973442713419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,2,128,1,float16,float16,0,0.6678720315297445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,2,128,1,fp8,fp8,0,0.6839839617411295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,1,128,1,float16,float16,0,0.6550506750742594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,2,128,1,float16,fp8,0,0.6638826529184977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,4,128,1,float16,float16,0,0.7070826689402262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,4,128,1,float16,fp8,0,0.6941013336181641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,1,128,1,float16,float16,0,0.33850133419036865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,4,128,1,fp8,fp8,0,0.7306613127390543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,16,128,1,float16,fp8,0,0.46769599119822186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,16,128,1,float16,float16,0,0.49371735254923504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,1,128,1,float16,fp8,0,0.3375999927520752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,16,128,1,fp8,fp8,0,0.40723200639088947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,8,128,1,float16,fp8,0,0.7531733512878418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,8,128,1,float16,float16,0,0.7664906978607178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,16,8,128,1,fp8,fp8,0,0.9207413196563721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,1,128,1,fp8,fp8,0,0.29155733187993366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,2,128,1,float16,float16,0,0.34641599655151367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,2,128,1,fp8,fp8,0,0.32264000177383423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,2,128,1,float16,fp8,0,0.34139732519785565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,4,128,1,float16,fp8,0,0.35277334849039715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,4,128,1,fp8,fp8,0,0.370037317276001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,4,128,1,float16,float16,0,0.3620213270187378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,8,128,1,float16,float16,0,0.5490080118179321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,8,128,1,float16,fp8,0,0.3851840098698934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,16,128,1,float16,float16,0,0.2542240023612976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,1,128,1,float16,float16,0,0.17971199750900269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,1,128,1,float16,fp8,0,0.1793066660563151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,16,8,128,1,fp8,fp8,0,0.38929065068562824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,1,128,1,fp8,fp8,0,0.16900267203648886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,16,128,1,float16,fp8,0,0.24307199319203696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,16,128,1,fp8,fp8,0,0.21076800425847372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,2,128,1,float16,float16,0,0.18146133422851562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,2,128,1,fp8,fp8,0,0.16556266943613687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,2,128,1,float16,fp8,0,0.18044267098108926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,4,128,1,float16,float16,0,0.1920213301976522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,4,128,1,float16,fp8,0,0.1879733403523763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,4,128,1,fp8,fp8,0,0.20522133509318033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,8,128,1,float16,float16,0,0.20628267526626587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,8,128,1,float16,fp8,0,0.20165866613388062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,16,128,1,float16,float16,0,0.13877333203951517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,1,128,1,float16,float16,0,0.1002506713072459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,1,128,1,float16,fp8,0,0.11293866237004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,16,128,1,fp8,fp8,0,0.1190720001856486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,16,128,1,float16,fp8,0,0.1329759955406189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,16,8,128,1,fp8,fp8,0,0.19851734240849814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,2,128,1,float16,float16,0,0.10156800349553426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,2,128,1,fp8,fp8,0,0.08868267138799031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,1,128,1,fp8,fp8,0,0.08649067083994548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,2,128,1,float16,fp8,0,0.10100266337394714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,4,128,1,float16,fp8,0,0.10530666510264079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,4,128,1,float16,float16,0,0.12995200355847678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,4,128,1,fp8,fp8,0,0.10002133250236511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,8,128,1,float16,fp8,0,0.11053867141405742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,8,128,1,float16,float16,0,0.11340266466140747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,1,128,1,float16,fp8,0,0.05783466498057047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,16,8,128,1,fp8,fp8,0,0.10486400127410889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,1,128,1,float16,float16,0,0.05913066864013672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,16,128,1,float16,float16,0,0.08238933483759563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,16,128,1,float16,fp8,0,0.07789866626262665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,1,128,1,fp8,fp8,0,0.04938666522502899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,2,128,1,float16,fp8,0,0.05909333129723867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,16,128,1,fp8,fp8,0,0.06613333523273468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,2,128,1,fp8,fp8,0,0.05043200155099233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,2,128,1,float16,float16,0,0.059024001161257424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,4,128,1,float16,float16,0,0.0650133341550827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,4,128,1,float16,fp8,0,0.06201600035031637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,4,128,1,fp8,fp8,0,0.05611200133959452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,8,128,1,float16,float16,0,0.0642986645301183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,16,128,1,fp8,fp8,0,0.043920000394185386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,16,128,1,float16,float16,0,0.0401706670721372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,1,128,1,float16,fp8,0,0.03437866767247518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,8,128,1,fp8,fp8,0,0.057258665561676025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,1,128,1,float16,float16,0,0.034416000048319496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,16,8,128,1,float16,fp8,0,0.0642080008983612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,16,128,1,float16,fp8,0,0.038704000413417816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,2,128,1,float16,float16,0,0.03514666606982549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,2,128,1,float16,fp8,0,0.035173334181308746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,2,128,1,fp8,fp8,0,0.03109866629044215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,1,128,1,fp8,fp8,0,0.030026666820049286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,4,128,1,fp8,fp8,0,0.03309333324432373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,4,128,1,float16,float16,0,0.036570665736993156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,8,128,1,float16,float16,0,0.03792533278465271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,4,128,1,float16,fp8,0,0.036362667878468834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,8,128,1,float16,fp8,0,0.0367253323396047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,16,128,1,float16,fp8,0,0.02496533344189326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,16,8,128,1,fp8,fp8,0,0.034287999073664345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,16,128,1,fp8,fp8,0,0.023247999449570973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,16,128,1,float16,float16,0,0.025349333882331848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,1,128,1,float16,fp8,0,0.023082666099071503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,1,128,1,float16,float16,0,0.023189333577950794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,2,128,1,float16,float16,0,0.023039999107519787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,2,128,1,fp8,fp8,0,0.0215786670645078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,2,128,1,float16,fp8,0,0.023658665517965954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,1,128,1,fp8,fp8,0,0.021221332252025604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,4,128,1,float16,float16,0,0.024495999018351238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,4,128,1,float16,fp8,0,0.02447466552257538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,4,128,1,fp8,fp8,0,0.02271466702222824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,8,128,1,float16,float16,0,0.024570666253566742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,8,128,1,fp8,fp8,0,0.022800001005331676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,16,128,1,float16,float16,0,0.020469332734743755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,16,8,128,1,float16,fp8,0,0.024666666984558105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,1,128,1,float16,float16,0,0.019354666272799175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,16,128,1,fp8,fp8,0,0.019088000059127808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,1,128,1,float16,fp8,0,0.01907733331123988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,16,128,1,float16,fp8,0,0.020661332954963047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,2,128,1,float16,float16,0,0.019391999890406925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,2,128,1,float16,fp8,0,0.019359999646743137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,2,128,1,fp8,fp8,0,0.018650667121013004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,4,128,1,float16,float16,0,0.019744000087181728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,4,128,1,float16,fp8,0,0.01982933282852173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,1,128,1,fp8,fp8,0,0.018485333770513535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,4,128,1,fp8,fp8,0,0.01911466692884763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,8,128,1,float16,fp8,0,0.020175999651352566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,8,128,1,fp8,fp8,0,0.01940800001223882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,16,8,128,1,float16,float16,0,0.019866666446129482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,16,128,1,float16,fp8,0,0.018954666952292126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,16,128,1,float16,float16,0,0.019093333433071773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,16,128,1,fp8,fp8,0,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,1,128,1,fp8,fp8,0,0.01736533393462499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,1,128,1,float16,float16,0,0.018191999445358913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,1,128,1,float16,fp8,0,0.018357332795858383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,2,128,1,float16,float16,0,0.018031999468803406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,2,128,1,fp8,fp8,0,0.017583999782800674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,4,128,1,float16,fp8,0,0.01848000039656957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,4,128,1,float16,float16,0,0.022133332987626392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,4,128,1,fp8,fp8,0,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,8,128,1,fp8,fp8,0,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,2,128,1,float16,fp8,0,0.018511999398469925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,8,128,1,float16,fp8,0,0.018543999642133713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,16,8,128,1,float16,float16,0,0.018320000420014065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,1,128,1,float16,float16,0,0.31941866874694824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,1,128,1,float16,fp8,0,0.31596267223358154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,1,128,1,fp8,fp8,0,0.3275466759999593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,2,128,1,float16,fp8,0,0.3216853340466817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,2,128,1,float16,float16,0,0.3291306694348653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,4,128,1,float16,float16,0,0.34539198875427246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,2,128,1,fp8,fp8,0,0.32356266180674237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,4,128,1,float16,fp8,0,0.3360053300857544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,4,128,1,fp8,fp8,0,0.3717973232269287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,8,128,1,float16,float16,0,0.3712799946467082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,16,128,1,float16,float16,0,0.262389341990153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,1,128,1,float16,float16,0,0.17092265685399374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,8,128,1,float16,fp8,0,0.5310133298238119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,16,8,128,1,fp8,fp8,0,0.38397332032521564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,16,128,1,float16,fp8,0,0.2553600072860718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,1,128,1,float16,fp8,0,0.1705120007197062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,16,128,1,fp8,fp8,0,0.21122666200002035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,1,128,1,fp8,fp8,0,0.15589333573977152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,2,128,1,float16,float16,0,0.17695466677347818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,2,128,1,fp8,fp8,0,0.18355733156204224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,4,128,1,float16,float16,0,0.18596800168355307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,2,128,1,float16,fp8,0,0.1732800006866455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,4,128,1,float16,fp8,0,0.1831573247909546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,4,128,1,fp8,fp8,0,0.1918026606241862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,8,128,1,float16,fp8,0,0.19513066609700522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,8,128,1,float16,float16,0,0.19882132609685263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,1,128,1,float16,float16,0,0.09769066174825032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,1,128,1,float16,fp8,0,0.09733333190282185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,16,8,128,1,fp8,fp8,0,0.19659199317296347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,16,128,1,float16,float16,0,0.20417600870132446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,16,128,1,float16,fp8,0,0.1365653375784556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,16,128,1,fp8,fp8,0,0.11288533608118693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,1,128,1,fp8,fp8,0,0.08688533306121826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,2,128,1,fp8,fp8,0,0.08819733063379924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,4,128,1,float16,float16,0,0.1067039966583252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,2,128,1,float16,float16,0,0.10106133421262105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,2,128,1,float16,fp8,0,0.09923199812571208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,4,128,1,fp8,fp8,0,0.0999679962793986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,4,128,1,float16,fp8,0,0.12709866960843405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,8,128,1,float16,fp8,0,0.11114133397738139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,8,128,1,float16,float16,0,0.11264000336329143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,1,128,1,float16,float16,0,0.0545653353134791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,16,8,128,1,fp8,fp8,0,0.10334933797518413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,16,128,1,float16,float16,0,0.07679999868075053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,1,128,1,float16,fp8,0,0.056133334835370384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,1,128,1,fp8,fp8,0,0.04916266600290934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,16,128,1,fp8,fp8,0,0.06393600006898244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,16,128,1,float16,fp8,0,0.07413333157698314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,2,128,1,float16,fp8,0,0.05579733351866404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,2,128,1,float16,float16,0,0.05588266750176748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,2,128,1,fp8,fp8,0,0.050554667909940086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,4,128,1,float16,float16,0,0.06016000111897787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,4,128,1,fp8,fp8,0,0.05604266623655955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,8,128,1,float16,fp8,0,0.061050668358802795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,8,128,1,fp8,fp8,0,0.0565226674079895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,16,128,1,float16,fp8,0,0.03827200084924698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,16,128,1,fp8,fp8,0,0.03771200031042099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,1,128,1,float16,float16,0,0.03315199911594391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,4,128,1,float16,fp8,0,0.06159466505050659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,16,8,128,1,float16,float16,0,0.06305066744486491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,16,128,1,float16,float16,0,0.03957866628964742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,1,128,1,fp8,fp8,0,0.030031998952229817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,1,128,1,float16,fp8,0,0.03366933266321818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,2,128,1,float16,float16,0,0.03412266572316488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,2,128,1,fp8,fp8,0,0.03081600119670232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,2,128,1,float16,fp8,0,0.03425066669782003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,4,128,1,float16,float16,0,0.03618666778008143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,4,128,1,float16,fp8,0,0.035061334570248924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,16,128,1,float16,float16,0,0.02442666639884313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,4,128,1,fp8,fp8,0,0.03282133241494497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,8,128,1,float16,fp8,0,0.0359946663180987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,8,128,1,float16,float16,0,0.036303999523321785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,16,8,128,1,fp8,fp8,0,0.03369600077470144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,16,128,1,float16,fp8,0,0.024288001159826916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,16,128,1,fp8,fp8,0,0.023061332603295643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,1,128,1,float16,float16,0,0.022416000564893086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,1,128,1,float16,fp8,0,0.02235200007756551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,2,128,1,float16,float16,0,0.02292799949645996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,2,128,1,float16,fp8,0,0.022954667607943218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,2,128,1,fp8,fp8,0,0.021354667842388153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,4,128,1,float16,fp8,0,0.023946667710940044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,4,128,1,float16,float16,0,0.02401600033044815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,1,128,1,fp8,fp8,0,0.020874666670958202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,4,128,1,fp8,fp8,0,0.022511998812357586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,8,128,1,float16,float16,0,0.02385066697994868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,8,128,1,float16,fp8,0,0.024480000138282776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,16,128,1,float16,float16,0,0.01905599981546402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,16,128,1,float16,fp8,0,0.019487999379634857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,16,8,128,1,fp8,fp8,0,0.022698665658632915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,16,128,1,fp8,fp8,0,0.019023999571800232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,1,128,1,float16,fp8,0,0.0182239996890227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,1,128,1,fp8,fp8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,1,128,1,float16,float16,0,0.018543999642133713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,2,128,1,float16,float16,0,0.018357332795858383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,2,128,1,float16,fp8,0,0.01846933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,2,128,1,fp8,fp8,0,0.018629333625237148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,4,128,1,float16,float16,0,0.018506667266289394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,8,128,1,float16,float16,0,0.018661333868900936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,4,128,1,float16,fp8,0,0.0191040001809597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,4,128,1,fp8,fp8,0,0.01907733331123988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,16,128,1,float16,float16,0,0.016938666502634685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,8,128,1,float16,fp8,0,0.01924266666173935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,16,128,1,float16,fp8,0,0.0173333336909612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,16,8,128,1,fp8,fp8,0,0.01926400015751521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,1,128,1,float16,float16,0,0.01681600014368693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,1,128,1,float16,fp8,0,0.01714133347074191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,16,128,1,fp8,fp8,0,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,1,128,1,fp8,fp8,0,0.01695466662446658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,2,128,1,float16,float16,0,0.016789333273967106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,2,128,1,fp8,fp8,0,0.017456000049908955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,8,128,1,float16,float16,0,0.01695999999841054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,4,128,1,fp8,fp8,0,0.0173333336909612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,4,128,1,float16,float16,0,0.016943999876578648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,2,128,1,float16,fp8,0,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,8,128,1,float16,fp8,0,0.017488000293572743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,4,128,1,float16,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,16,128,1,float16,float16,0,0.01637866720557213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,16,8,128,1,fp8,fp8,0,0.017450666675964992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,16,128,1,float16,fp8,0,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,16,128,1,fp8,fp8,0,0.016778666526079178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,1,128,1,float16,float16,0,0.016447999825080235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,1,128,1,fp8,fp8,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,1,128,1,float16,fp8,0,0.016858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,2,128,1,float16,fp8,0,0.01701333373785019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,2,128,1,float16,float16,0,0.01623999948302905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,4,128,1,float16,float16,0,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,2,128,1,fp8,fp8,0,0.016789333273967106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,4,128,1,float16,fp8,0,0.016997333616018295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,4,128,1,fp8,fp8,0,0.01691199963291486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,8,128,1,float16,float16,0,0.016389333953460056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,8,128,1,fp8,fp8,0,0.016976000120242436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,16,8,128,1,float16,fp8,0,0.01691199963291486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,1,128,1,float16,float16,0,0.19825067122777304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,1,128,1,float16,fp8,0,0.19824000199635824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,2,128,1,float16,float16,0,0.20620266596476236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,1,128,1,fp8,fp8,0,0.20465066035588583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,2,128,1,float16,fp8,0,0.20036800702412924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,2,128,1,fp8,fp8,0,0.21427732706069946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,4,128,1,float16,float16,0,0.27292267481486004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,4,128,1,float16,fp8,0,0.21141332387924194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,4,128,1,fp8,fp8,0,0.2380746603012085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,8,128,1,float16,float16,0,0.22927467028299967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,16,128,1,float16,float16,0,0.13963199655214945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,8,128,1,fp8,fp8,0,0.24597867329915366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,16,8,128,1,float16,fp8,0,0.22425599892934164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,16,128,1,float16,fp8,0,0.13317333658536276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,16,128,1,fp8,fp8,0,0.16587733229001364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,1,128,1,float16,fp8,0,0.1088746686776479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,1,128,1,fp8,fp8,0,0.11249066392580669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,1,128,1,float16,float16,0,0.10972799857457478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,2,128,1,float16,float16,0,0.11296533544858296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,2,128,1,float16,fp8,0,0.11171733339627583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,4,128,1,float16,fp8,0,0.1435466706752777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,2,128,1,fp8,fp8,0,0.11523733536402385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,4,128,1,fp8,fp8,0,0.12626133362452188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,4,128,1,float16,float16,0,0.12050666411717732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,8,128,1,float16,float16,0,0.12665599584579468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,8,128,1,float16,fp8,0,0.12293333808581035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,16,128,1,float16,float16,0,0.07452799876530965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,16,8,128,1,fp8,fp8,0,0.12794666488965353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,1,128,1,float16,float16,0,0.060122668743133545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,16,128,1,float16,fp8,0,0.09575466314951579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,1,128,1,float16,fp8,0,0.06038933495680491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,1,128,1,fp8,fp8,0,0.06329066554705302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,16,128,1,fp8,fp8,0,0.07635733485221863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,2,128,1,fp8,fp8,0,0.06491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,4,128,1,float16,float16,0,0.06472533444563548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,2,128,1,float16,float16,0,0.06198933223883311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,4,128,1,float16,fp8,0,0.06657599906126659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,2,128,1,float16,fp8,0,0.061664000153541565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,8,128,1,float16,float16,0,0.0682666649421056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,4,128,1,fp8,fp8,0,0.06986133257548015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,8,128,1,float16,fp8,0,0.06723733246326447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,16,8,128,1,fp8,fp8,0,0.07062933345635732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,1,128,1,float16,float16,0,0.03589866558710734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,16,128,1,float16,float16,0,0.04030400017897288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,16,128,1,fp8,fp8,0,0.04933333396911621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,16,128,1,float16,fp8,0,0.03908800085385641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,1,128,1,float16,fp8,0,0.03598399957021078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,1,128,1,fp8,fp8,0,0.037087999284267426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,2,128,1,float16,float16,0,0.036474667489528656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,2,128,1,fp8,fp8,0,0.03775466730197271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,4,128,1,float16,float16,0,0.03751999884843826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,4,128,1,float16,fp8,0,0.03800000001986822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,2,128,1,float16,fp8,0,0.036330667634805046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,8,128,1,fp8,fp8,0,0.041493333876132965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,8,128,1,float16,float16,0,0.0384853333234787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,4,128,1,fp8,fp8,0,0.03990933299064636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,16,8,128,1,float16,fp8,0,0.038773333032925926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,16,128,1,float16,float16,0,0.025290665527184803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,16,128,1,float16,fp8,0,0.02499199906984965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,1,128,1,float16,fp8,0,0.024101334313551586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,16,128,1,fp8,fp8,0,0.02632533262173335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,1,128,1,float16,float16,0,0.02401600033044815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,1,128,1,fp8,fp8,0,0.024400000770886738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,2,128,1,fp8,fp8,0,0.0249439999461174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,4,128,1,float16,fp8,0,0.02533866713444392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,2,128,1,float16,float16,0,0.024149333437283833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,4,128,1,fp8,fp8,0,0.025962665677070618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,4,128,1,float16,float16,0,0.02462933212518692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,2,128,1,float16,fp8,0,0.02399466683467229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,8,128,1,float16,fp8,0,0.025487999121348064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,16,128,1,float16,float16,0,0.019039999693632126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,8,128,1,fp8,fp8,0,0.026005332668622334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,16,8,128,1,float16,float16,0,0.02550400048494339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,16,128,1,fp8,fp8,0,0.01926400015751521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,1,128,1,float16,float16,0,0.017466666797796886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,1,128,1,fp8,fp8,0,0.0182239996890227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,16,128,1,float16,fp8,0,0.018853332847356796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,2,128,1,float16,float16,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,2,128,1,float16,fp8,0,0.017664000391960144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,1,128,1,float16,fp8,0,0.017525333911180496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,4,128,1,float16,float16,0,0.018277333428462345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,4,128,1,fp8,fp8,0,0.019146667172511418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,2,128,1,fp8,fp8,0,0.018677332748969395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,4,128,1,float16,fp8,0,0.01801066721479098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,8,128,1,float16,fp8,0,0.01868266612291336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,8,128,1,fp8,fp8,0,0.019141333798567455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,16,8,128,1,float16,float16,0,0.018650667121013004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,16,128,1,float16,float16,0,0.016010666886965435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,16,128,1,float16,fp8,0,0.01640533283352852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,16,128,1,fp8,fp8,0,0.01720000058412552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,1,128,1,float16,float16,0,0.01580799991885821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,1,128,1,float16,fp8,0,0.0161920003592968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,2,128,1,float16,float16,0,0.015834666788578033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,1,128,1,fp8,fp8,0,0.016895999511082966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,4,128,1,float16,fp8,0,0.01626666635274887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,4,128,1,float16,float16,0,0.015824000040690105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,2,128,1,fp8,fp8,0,0.017279999951521557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,4,128,1,fp8,fp8,0,0.01752000053723653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,2,128,1,float16,fp8,0,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,8,128,1,float16,float16,0,0.01587733378012975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,8,128,1,fp8,fp8,0,0.01754666616519292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,16,128,1,float16,float16,0,0.015376000354687372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,16,8,128,1,float16,fp8,0,0.019258666783571243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,16,128,1,float16,fp8,0,0.015589332828919092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,1,128,1,float16,float16,0,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,2,128,1,float16,float16,0,0.015253332753976187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,16,128,1,fp8,fp8,0,0.016271999726692837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,1,128,1,fp8,fp8,0,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,2,128,1,float16,fp8,0,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,1,128,1,float16,fp8,0,0.01590399940808614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,4,128,1,float16,float16,0,0.015263999501864115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,2,128,1,fp8,fp8,0,0.016496000190575916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,4,128,1,float16,fp8,0,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,4,128,1,fp8,fp8,0,0.01648533344268799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,8,128,1,float16,float16,0,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,8,128,1,float16,fp8,0,0.015829333414634068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,16,8,128,1,fp8,fp8,0,0.016586666305859882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,16,128,1,float16,fp8,0,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,16,128,1,fp8,fp8,0,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,1,128,1,float16,float16,0,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,1,128,1,float16,fp8,0,0.015610666324694952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,16,128,1,float16,float16,0,0.015008000036080679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,2,128,1,float16,float16,0,0.014885333677132925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,2,128,1,float16,fp8,0,0.015040000279744467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,1,128,1,fp8,fp8,0,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,2,128,1,fp8,fp8,0,0.016309333344300587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,4,128,1,float16,fp8,0,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,4,128,1,float16,float16,0,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,4,128,1,fp8,fp8,0,0.0162773331006368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,8,128,1,fp8,fp8,0,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,8,128,1,float16,fp8,0,0.015333333363135656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,16,8,128,1,float16,float16,0,0.01482133318980535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,1,128,1,float16,fp8,0,0.153738667567571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,1,128,1,float16,float16,0,0.14124799768129984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,2,128,1,float16,float16,0,0.14547733465830484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,2,128,1,float16,fp8,0,0.1444960037867228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,1,128,1,fp8,fp8,0,0.16735466321309408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,2,128,1,fp8,fp8,0,0.1688106656074524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,4,128,1,float16,float16,0,0.15043733517328897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,4,128,1,fp8,fp8,0,0.18263999621073404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,4,128,1,float16,fp8,0,0.14828800161679587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,16,128,1,float16,float16,0,0.09076799949010213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,8,128,1,float16,fp8,0,0.1541706621646881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,8,128,1,fp8,fp8,0,0.1795519987742106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,16,128,1,float16,fp8,0,0.08770133058230083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,16,8,128,1,float16,float16,0,0.15825066963831583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,16,128,1,fp8,fp8,0,0.10518933335940044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,1,128,1,float16,float16,0,0.07746133208274841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,1,128,1,float16,fp8,0,0.07696533203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,1,128,1,fp8,fp8,0,0.0916373332341512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,2,128,1,float16,fp8,0,0.0767680009206136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,2,128,1,fp8,fp8,0,0.09263466795285542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,2,128,1,float16,float16,0,0.07711466650168101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,4,128,1,float16,fp8,0,0.07985066870848338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,4,128,1,float16,float16,0,0.07983466486136119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,4,128,1,fp8,fp8,0,0.10036266843477885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,8,128,1,float16,fp8,0,0.08336533109347026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,8,128,1,float16,float16,0,0.08422399560610454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,16,128,1,float16,float16,0,0.0468800018231074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,1,128,1,float16,fp8,0,0.04400533437728882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,1,128,1,float16,float16,0,0.043098668257395424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,16,128,1,fp8,fp8,0,0.0587360014518102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,16,8,128,1,fp8,fp8,0,0.09824533263842265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,16,128,1,float16,fp8,0,0.04537599782148997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,1,128,1,fp8,fp8,0,0.05102399984995524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,2,128,1,float16,fp8,0,0.04338666796684265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,4,128,1,float16,fp8,0,0.04513599971930186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,2,128,1,fp8,fp8,0,0.051632001996040344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,4,128,1,fp8,fp8,0,0.05389333268006643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,2,128,1,float16,float16,0,0.043605332573254905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,4,128,1,float16,float16,0,0.04518933097521464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,8,128,1,float16,float16,0,0.045109331607818604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,8,128,1,float16,fp8,0,0.0459146648645401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,16,8,128,1,fp8,fp8,0,0.05495999753475189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,16,128,1,float16,fp8,0,0.027930667002995808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,16,128,1,float16,float16,0,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,16,128,1,fp8,fp8,0,0.0329066663980484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,1,128,1,fp8,fp8,0,0.03166933357715607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,1,128,1,float16,fp8,0,0.02739199995994568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,1,128,1,float16,float16,0,0.02719466636578242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,2,128,1,float16,float16,0,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,2,128,1,float16,fp8,0,0.027429332335789997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,2,128,1,fp8,fp8,0,0.032170665760835014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,4,128,1,float16,float16,0,0.028277332584063213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,4,128,1,float16,fp8,0,0.028416000306606293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,4,128,1,fp8,fp8,0,0.032885332902272545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,8,128,1,float16,float16,0,0.028346667687098186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,8,128,1,fp8,fp8,0,0.03309333324432373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,16,128,1,float16,float16,0,0.01982933282852173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,16,128,1,float16,fp8,0,0.0201706662774086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,16,128,1,fp8,fp8,0,0.023120000958442688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,16,8,128,1,float16,fp8,0,0.028330666323502857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,1,128,1,float16,float16,0,0.019194666296243668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,1,128,1,float16,fp8,0,0.018853332847356796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,2,128,1,float16,float16,0,0.018810667097568512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,2,128,1,float16,fp8,0,0.019152000546455383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,4,128,1,float16,float16,0,0.019066666563351948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,1,128,1,fp8,fp8,0,0.0215786670645078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,2,128,1,fp8,fp8,0,0.022122666239738464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,4,128,1,fp8,fp8,0,0.02274133265018463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,8,128,1,float16,float16,0,0.019845332950353622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,4,128,1,float16,fp8,0,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,8,128,1,fp8,fp8,0,0.022485333184401195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,16,8,128,1,float16,fp8,0,0.019765333582957584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,16,128,1,float16,float16,0,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,1,128,1,float16,float16,0,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,16,128,1,fp8,fp8,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,1,128,1,float16,fp8,0,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,16,128,1,float16,fp8,0,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,2,128,1,float16,float16,0,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,2,128,1,fp8,fp8,0,0.01770666614174843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,2,128,1,float16,fp8,0,0.01578666642308235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,1,128,1,fp8,fp8,0,0.017466666797796886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,4,128,1,float16,float16,0,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,4,128,1,float16,fp8,0,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,4,128,1,fp8,fp8,0,0.017994667092959087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,8,128,1,float16,float16,0,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,8,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,16,8,128,1,fp8,fp8,0,0.018021332720915478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,16,128,1,float16,float16,0,0.014933332800865173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,1,128,1,float16,float16,0,0.014645333091417948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,16,128,1,fp8,fp8,0,0.01724799970785777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,1,128,1,float16,fp8,0,0.014970666418472925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,16,128,1,float16,fp8,0,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,1,128,1,fp8,fp8,0,0.016666666915019352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,2,128,1,float16,float16,0,0.014565333724021912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,2,128,1,float16,fp8,0,0.015173333386580149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,4,128,1,float16,float16,0,0.014581333845853806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,2,128,1,fp8,fp8,0,0.016773333152135212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,4,128,1,fp8,fp8,0,0.016837333639462788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,4,128,1,float16,fp8,0,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,8,128,1,fp8,fp8,0,0.017082666357358296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,8,128,1,float16,fp8,0,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,16,128,1,float16,fp8,0,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,16,128,1,float16,float16,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,16,8,128,1,float16,float16,0,0.014554666976133982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,1,128,1,float16,float16,0,0.014245333770910898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,16,128,1,fp8,fp8,0,0.016208000481128693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,1,128,1,float16,fp8,0,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,1,128,1,fp8,fp8,0,0.016063999384641647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,2,128,1,float16,fp8,0,0.014864000181357065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,4,128,1,float16,fp8,0,0.014938666174809137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,2,128,1,fp8,fp8,0,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,4,128,1,fp8,fp8,0,0.01658133293191592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,2,128,1,float16,float16,0,0.014138666292031607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,4,128,1,float16,float16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,8,128,1,float16,float16,0,0.0144213338692983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,8,128,1,fp8,fp8,0,0.01651200031240781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,16,8,128,1,float16,fp8,0,0.014602666099866232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,16,128,1,float16,float16,0,0.013951999445756277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,16,128,1,fp8,fp8,0,0.016117333124081295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,16,128,1,float16,fp8,0,0.014458666245142618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,1,128,1,float16,float16,0,0.013951999445756277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,2,128,1,float16,float16,0,0.01423466702302297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,1,128,1,fp8,fp8,0,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,2,128,1,float16,fp8,0,0.014368000129858652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,1,128,1,float16,fp8,0,0.014815999815861383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,4,128,1,float16,float16,0,0.014058666924635569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,2,128,1,fp8,fp8,0,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,8,128,1,float16,float16,0,0.014293332894643148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,4,128,1,float16,fp8,0,0.01462399959564209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,8,128,1,float16,fp8,0,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,8,128,1,fp8,fp8,0,0.01597333326935768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,16,4,128,1,fp8,fp8,0,0.016021333634853363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,1,128,1,float16,float16,0,0.11923733353614807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,1,128,1,float16,fp8,0,0.11970133582750957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,2,128,1,float16,float16,0,0.12173333764076233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,2,128,1,float16,fp8,0,0.12035733461380005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,1,128,1,fp8,fp8,0,0.1415733297665914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,4,128,1,float16,float16,0,0.12175466616948445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,2,128,1,fp8,fp8,0,0.14330666263898215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,4,128,1,float16,fp8,0,0.12151466806729634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,8,128,1,float16,fp8,0,0.13062933087348938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,16,128,1,float16,float16,0,0.06467733283837636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,8,128,1,fp8,fp8,0,0.15132799744606018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,16,128,1,float16,fp8,0,0.061674664417902626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,4,128,1,fp8,fp8,0,0.1482133368651072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,16,8,128,1,float16,float16,0,0.12586667140324911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,16,128,1,fp8,fp8,0,0.0851146678129832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,1,128,1,float16,float16,0,0.06497600177923839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,1,128,1,float16,fp8,0,0.06527466575304668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,1,128,1,fp8,fp8,0,0.07849066456158955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,2,128,1,float16,float16,0,0.06562133133411407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,4,128,1,float16,fp8,0,0.06684799989064534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,4,128,1,float16,float16,0,0.06726400057474773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,2,128,1,float16,fp8,0,0.06485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,2,128,1,fp8,fp8,0,0.07982933521270752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,4,128,1,fp8,fp8,0,0.08136000235875447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,8,128,1,float16,float16,0,0.06825600067774455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,8,128,1,fp8,fp8,0,0.08319999774297078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,16,8,128,1,float16,fp8,0,0.06709866722424825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,16,128,1,float16,float16,0,0.036805334190527596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,16,128,1,float16,fp8,0,0.03639466563860575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,1,128,1,fp8,fp8,0,0.04549333453178406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,16,128,1,fp8,fp8,0,0.0465280016263326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,2,128,1,float16,float16,0,0.03881066789229711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,1,128,1,float16,float16,0,0.0384853333234787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,2,128,1,float16,fp8,0,0.038480001191298165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,2,128,1,fp8,fp8,0,0.04608533283074697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,1,128,1,float16,fp8,0,0.03867733230193456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,4,128,1,float16,float16,0,0.039493332306543984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,4,128,1,float16,fp8,0,0.03961066653331121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,8,128,1,float16,float16,0,0.03959999978542328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,8,128,1,fp8,fp8,0,0.04713066418965658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,4,128,1,fp8,fp8,0,0.04726399978001913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,16,128,1,float16,float16,0,0.024490666886170704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,16,128,1,fp8,fp8,0,0.03009066730737686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,16,8,128,1,float16,fp8,0,0.039861333866914116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,1,128,1,float16,float16,0,0.024858665963013966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,16,128,1,float16,fp8,0,0.024538666009902954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,1,128,1,float16,fp8,0,0.025029333929220837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,1,128,1,fp8,fp8,0,0.028970666229724884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,2,128,1,float16,float16,0,0.024773334463437397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,2,128,1,float16,fp8,0,0.024725332856178284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,2,128,1,fp8,fp8,0,0.029648000995318096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,4,128,1,float16,fp8,0,0.025744001070658367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,8,128,1,float16,float16,0,0.02515733242034912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,4,128,1,float16,float16,0,0.02481599897146225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,4,128,1,fp8,fp8,0,0.02958400050799052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,8,128,1,fp8,fp8,0,0.029674666623274486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,16,8,128,1,float16,fp8,0,0.025055999557177227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,16,128,1,float16,fp8,0,0.018800000349680584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,16,128,1,float16,float16,0,0.01817600056529045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,16,128,1,fp8,fp8,0,0.021695998807748158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,1,128,1,float16,float16,0,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,2,128,1,fp8,fp8,0,0.021344001094500225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,4,128,1,float16,float16,0,0.018058666338523228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,1,128,1,fp8,fp8,0,0.02107733239730199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,4,128,1,float16,fp8,0,0.018309333672126133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,2,128,1,float16,float16,0,0.01781333362062772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,2,128,1,float16,fp8,0,0.018191999445358913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,1,128,1,float16,fp8,0,0.017909333109855652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,4,128,1,fp8,fp8,0,0.020997333029905956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,8,128,1,float16,float16,0,0.018170667191346485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,8,128,1,float16,fp8,0,0.018458666900793713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,16,8,128,1,fp8,fp8,0,0.021589333812395733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,16,128,1,float16,float16,0,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,16,128,1,float16,fp8,0,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,1,128,1,float16,fp8,0,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,16,128,1,fp8,fp8,0,0.017504000415404636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,1,128,1,float16,float16,0,0.015008000036080679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,1,128,1,fp8,fp8,0,0.017093333105246227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,2,128,1,float16,float16,0,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,4,128,1,float16,fp8,0,0.015498666713635126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,2,128,1,fp8,fp8,0,0.01754133279124896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,2,128,1,float16,fp8,0,0.015402667224407196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,4,128,1,float16,float16,0,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,4,128,1,fp8,fp8,0,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,8,128,1,float16,fp8,0,0.015647999942302704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,8,128,1,float16,float16,0,0.015418666104475657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,16,128,1,float16,float16,0,0.014554666976133982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,16,128,1,float16,fp8,0,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,16,8,128,1,fp8,fp8,0,0.017952000101407368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,1,128,1,float16,float16,0,0.014202666779359182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,16,128,1,fp8,fp8,0,0.016421332955360413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,1,128,1,float16,fp8,0,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,1,128,1,fp8,fp8,0,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,2,128,1,float16,fp8,0,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,2,128,1,fp8,fp8,0,0.01661866654952367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,4,128,1,float16,float16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,4,128,1,float16,fp8,0,0.01452800010641416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,8,128,1,float16,float16,0,0.014186666657527288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,2,128,1,float16,float16,0,0.014485333114862442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,8,128,1,float16,fp8,0,0.014949332922697067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,4,128,1,fp8,fp8,0,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,16,8,128,1,fp8,fp8,0,0.016864000509182613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,16,128,1,float16,float16,0,0.014197333405415217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,16,128,1,float16,fp8,0,0.014762666076421738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,16,128,1,fp8,fp8,0,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,1,128,1,float16,float16,0,0.014080000420411428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,2,128,1,float16,fp8,0,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,2,128,1,float16,float16,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,1,128,1,fp8,fp8,0,0.016410666207472484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,1,128,1,float16,fp8,0,0.014805333067973455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,2,128,1,fp8,fp8,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,4,128,1,float16,float16,0,0.014469332993030548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,4,128,1,float16,fp8,0,0.014682666709025701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,4,128,1,fp8,fp8,0,0.01640533283352852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,8,128,1,float16,float16,0,0.01414399966597557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,8,128,1,float16,fp8,0,0.014762666076421738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,16,128,1,float16,fp8,0,0.014490666488806406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,16,8,128,1,fp8,fp8,0,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,16,128,1,fp8,fp8,0,0.01584533353646596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,1,128,1,float16,float16,0,0.013797332843144735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,1,128,1,float16,fp8,0,0.01461333284775416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,16,128,1,float16,float16,0,0.01394133393963178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,2,128,1,float16,float16,0,0.013989333063364029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,1,128,1,fp8,fp8,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,4,128,1,float16,float16,0,0.013957332819700241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,2,128,1,float16,fp8,0,0.014602666099866232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,2,128,1,fp8,fp8,0,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,4,128,1,float16,fp8,0,0.014282666146755219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,4,128,1,fp8,fp8,0,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,8,128,1,float16,fp8,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,8,128,1,fp8,fp8,0,0.016042667130629223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,16,8,128,1,float16,float16,0,0.014111999422311783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,1,128,1,float16,fp8,0,12.46127446492513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,1,128,1,float16,float16,0,12.362069447835287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,1,128,1,fp8,fp8,0,7.775530497233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,2,128,1,float16,float16,0,12.51370112101237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,2,128,1,float16,fp8,0,12.202752431233725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,2,128,1,fp8,fp8,0,7.828789393107097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,4,128,1,float16,float16,0,12.338741302490234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,4,128,1,float16,fp8,0,12.365578969319662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,1,128,1,float16,float16,0,6.131914774576823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,1,128,1,float16,fp8,0,6.046522776285808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,1,128,1,fp8,fp8,0,3.852661450703939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,2,128,1,float16,float16,0,5.950970967610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,12,4,128,1,fp8,fp8,0,8.10102399190267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,12,128,1,fp8,fp8,0,3.963786760965983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,12,128,1,float16,float16,0,5.914405186971028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,12,128,1,float16,fp8,0,5.82583491007487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,2,128,1,fp8,fp8,0,3.945530573527018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,2,128,1,float16,fp8,0,6.153952280680339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,4,128,1,float16,float16,0,6.176026662190755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,1,128,1,float16,float16,0,2.985738754272461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,4,128,1,float16,fp8,0,5.811578750610352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,12,128,1,float16,float16,0,3.0283466974894204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,12,4,128,1,fp8,fp8,0,4.009242693583171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,12,128,1,float16,fp8,0,2.9257920583089194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,1,128,1,float16,fp8,0,3.004410743713379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,1,128,1,fp8,fp8,0,2.0085652669270835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,12,128,1,fp8,fp8,0,1.972325325012207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,2,128,1,float16,float16,0,2.9886773427327475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,2,128,1,fp8,fp8,0,2.022650718688965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,4,128,1,float16,float16,0,3.0502452850341797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,2,128,1,float16,fp8,0,2.9804000854492188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,12,128,1,float16,float16,0,1.5703840255737305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,4,128,1,float16,fp8,0,3.029792149861654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,12,128,1,float16,fp8,0,1.5530986785888672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,12,4,128,1,fp8,fp8,0,2.026064078013102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,1,128,1,float16,fp8,0,1.5929759343465169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,1,128,1,float16,float16,0,1.6230826377868652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,2,128,1,float16,float16,0,1.6089386940002441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,2,128,1,fp8,fp8,0,1.0320160388946533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,2,128,1,float16,fp8,0,1.5957973798116047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,12,128,1,fp8,fp8,0,1.0109600226084392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,4,128,1,float16,float16,0,1.6031146049499512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,1,128,1,fp8,fp8,0,1.029088020324707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,4,128,1,float16,fp8,0,1.6034132639567058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,12,4,128,1,fp8,fp8,0,1.0428106784820557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,1,128,1,float16,float16,0,7.1586456298828125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,1,128,1,fp8,fp8,0,4.596703847249349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,1,128,1,float16,fp8,0,7.162495930989583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,2,128,1,float16,float16,0,7.111493428548177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,2,128,1,fp8,fp8,0,4.469018618265788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,2,128,1,float16,fp8,0,7.013189315795898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,4,128,1,float16,float16,0,7.174655914306641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,4,128,1,float16,fp8,0,7.185712178548177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,1,128,1,float16,fp8,0,3.369237263997396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,1,128,1,float16,float16,0,3.4062827428181968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,1,128,1,fp8,fp8,0,2.340288003285726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,12,4,128,1,fp8,fp8,0,4.532704035441081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,12,128,1,float16,float16,0,3.355274518330892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,12,128,1,fp8,fp8,0,2.281434694925944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,12,128,1,float16,fp8,0,3.4444907506306968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,2,128,1,float16,float16,0,3.485957463582357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,2,128,1,float16,fp8,0,3.364757219950358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,2,128,1,fp8,fp8,0,2.30621337890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,1,128,1,float16,float16,0,1.7706400553385417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,4,128,1,float16,float16,0,3.5869067509969077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,1,128,1,float16,fp8,0,1.7650559743245442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,4,128,1,float16,fp8,0,3.436848004659017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,12,128,1,float16,float16,0,1.7433120409647624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,12,128,1,fp8,fp8,0,1.2176000277201335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,12,128,1,float16,fp8,0,1.7367626825968425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,12,4,128,1,fp8,fp8,0,2.323925336201986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,1,128,1,fp8,fp8,0,1.2139306863149006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,2,128,1,float16,float16,0,1.8001227378845215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,2,128,1,fp8,fp8,0,1.2197439670562744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,2,128,1,float16,fp8,0,1.7695679664611816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,4,128,1,float16,fp8,0,1.797375996907552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,4,128,1,float16,float16,0,1.7820533116658528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,12,4,128,1,fp8,fp8,0,1.2270932992299397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,12,128,1,float16,float16,0,0.9553066889444987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,12,128,1,float16,fp8,0,0.9430879751841227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,1,128,1,float16,float16,0,0.9760053157806396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,1,128,1,fp8,fp8,0,0.6267093420028687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,12,128,1,fp8,fp8,0,0.640773336092631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,1,128,1,float16,fp8,0,0.9680853684743246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,2,128,1,float16,float16,0,0.9786187012990316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,2,128,1,float16,fp8,0,0.9684800306955973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,2,128,1,fp8,fp8,0,0.6286453406016032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,4,128,1,float16,float16,0,0.978874683380127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,4,128,1,float16,fp8,0,0.9725600083669027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,12,4,128,1,fp8,fp8,0,0.6360373497009277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,1,128,1,float16,float16,0,5.040666580200195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,1,128,1,float16,fp8,0,4.761397361755371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,1,128,1,fp8,fp8,0,3.2229865392049155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,2,128,1,float16,float16,0,4.9800106684366865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,2,128,1,float16,fp8,0,4.798442522684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,2,128,1,fp8,fp8,0,3.230229377746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,4,128,1,float16,float16,0,4.842410723368327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,4,128,1,float16,fp8,0,5.062005360921224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,1,128,1,float16,fp8,0,2.402714729309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,1,128,1,float16,float16,0,2.4551199277242026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,1,128,1,fp8,fp8,0,1.6666666666666667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,12,4,128,1,fp8,fp8,0,3.37825075785319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,2,128,1,float16,float16,0,2.4812960624694824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,12,128,1,float16,fp8,0,2.3895413080851235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,12,128,1,float16,float16,0,2.424880027770996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,2,128,1,float16,fp8,0,2.4101012547810874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,2,128,1,fp8,fp8,0,1.6725385983784993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,12,128,1,fp8,fp8,0,1.6697012583414714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,1,128,1,float16,float16,0,1.2856746514638264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,4,128,1,float16,float16,0,2.4598719278971353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,4,128,1,float16,fp8,0,2.474181334177653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,12,128,1,float16,float16,0,1.2668000062306721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,12,4,128,1,fp8,fp8,0,1.6952853202819824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,1,128,1,float16,fp8,0,1.2721386750539143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,12,128,1,fp8,fp8,0,0.887493371963501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,12,128,1,float16,fp8,0,1.2642133235931396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,1,128,1,fp8,fp8,0,0.8920586903889974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,2,128,1,float16,float16,0,1.291040023167928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,2,128,1,float16,fp8,0,1.2893813451131184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,2,128,1,fp8,fp8,0,0.8937439918518066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,4,128,1,float16,float16,0,1.3101759751637776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,4,128,1,float16,fp8,0,1.2811840375264485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,12,4,128,1,fp8,fp8,0,0.9013120333353678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,12,128,1,float16,float16,0,0.7047839959462484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,12,128,1,float16,fp8,0,0.6990666389465332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,1,128,1,float16,float16,0,0.7230133215586344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,1,128,1,float16,fp8,0,0.7113706270853678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,1,128,1,fp8,fp8,0,0.4618613322575887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,12,128,1,fp8,fp8,0,0.46003198623657227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,2,128,1,float16,fp8,0,0.7120107014973959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,2,128,1,float16,float16,0,0.7208906809488932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,2,128,1,fp8,fp8,0,0.46270934740702313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,4,128,1,float16,fp8,0,0.7204960187276205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,4,128,1,float16,float16,0,0.7225706577301025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,12,4,128,1,fp8,fp8,0,0.4675360123316447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,1,128,1,float16,float16,0,6.53212292989095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,1,128,1,float16,fp8,0,6.636266708374023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,1,128,1,fp8,fp8,0,4.264730771382649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,2,128,1,fp8,fp8,0,4.515237490336101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,2,128,1,float16,float16,0,6.5629119873046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,2,128,1,float16,fp8,0,6.474373499552409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,4,128,1,float16,float16,0,6.653493245442708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,4,128,1,float16,fp8,0,6.483066558837891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,1,128,1,float16,float16,0,3.2463626861572266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,1,128,1,float16,fp8,0,3.1444905598958335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,1,128,1,fp8,fp8,0,2.1846240361531577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,12,128,1,float16,float16,0,3.1895360946655273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,2,128,1,float16,float16,0,3.176032066345215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,12,128,1,float16,fp8,0,3.1234985987345376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,12,4,128,1,fp8,fp8,0,4.492954572041829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,12,128,1,fp8,fp8,0,2.3256053924560547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,2,128,1,float16,fp8,0,3.228384017944336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,2,128,1,fp8,fp8,0,2.1970826784769693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,4,128,1,float16,fp8,0,3.2435359954833984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,1,128,1,float16,float16,0,1.6306986808776855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,4,128,1,float16,float16,0,3.2240320841471353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,1,128,1,float16,fp8,0,1.6171092987060547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,12,128,1,float16,float16,0,1.6225706736246746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,12,4,128,1,fp8,fp8,0,2.2137014071146646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,12,128,1,fp8,fp8,0,1.198746681213379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,12,128,1,float16,fp8,0,1.6431199709574382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,1,128,1,fp8,fp8,0,1.135696013768514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,2,128,1,float16,float16,0,1.6353227297465007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,2,128,1,float16,fp8,0,1.6190026601155598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,2,128,1,fp8,fp8,0,1.145477294921875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,4,128,1,float16,float16,0,1.6418399810791016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,4,128,1,float16,fp8,0,1.6249759991963704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,12,128,1,float16,float16,0,0.8787840207417806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,12,4,128,1,fp8,fp8,0,1.1790186564127605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,1,128,1,float16,float16,0,0.8766986529032389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,12,128,1,float16,fp8,0,0.862885316212972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,1,128,1,fp8,fp8,0,0.5847466786702474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,1,128,1,float16,fp8,0,0.8683786392211914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,2,128,1,float16,float16,0,0.8794399897257487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,12,128,1,fp8,fp8,0,0.5986613432566324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,2,128,1,float16,fp8,0,0.8773653507232666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,2,128,1,fp8,fp8,0,0.5967893203099569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,4,128,1,float16,float16,0,0.8822346528371176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,4,128,1,float16,fp8,0,0.8734133243560791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,1,128,1,float16,fp8,0,0.4307466745376587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,1,128,1,float16,float16,0,0.43561065196990967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,12,4,128,1,fp8,fp8,0,0.5955413182576498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,12,128,1,float16,float16,0,0.43666664759318036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,12,128,1,float16,fp8,0,0.4288320144017537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,12,128,1,fp8,fp8,0,0.3251039981842041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,1,128,1,fp8,fp8,0,0.32233067353566486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,2,128,1,float16,float16,0,0.4359946648279826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,2,128,1,float16,fp8,0,0.4314826726913452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,4,128,1,float16,float16,0,0.4373386700948079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,4,128,1,fp8,fp8,0,0.32763199011484784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,2,128,1,fp8,fp8,0,0.3238826592763265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,12,4,128,1,float16,fp8,0,0.432645320892334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,1,128,1,float16,float16,0,3.7344799041748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,1,128,1,float16,fp8,0,3.815098762512207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,1,128,1,fp8,fp8,0,2.6138453483581543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,2,128,1,float16,float16,0,3.945232073465983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,2,128,1,float16,fp8,0,3.689674695332845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,2,128,1,fp8,fp8,0,2.627471923828125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,4,128,1,float16,fp8,0,3.741877237955729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,4,128,1,float16,float16,0,3.8867626190185547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,1,128,1,float16,float16,0,1.8981013298034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,1,128,1,float16,fp8,0,1.8718934059143066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,12,128,1,float16,fp8,0,1.9116692543029785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,12,128,1,float16,float16,0,1.9288105964660645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,12,128,1,fp8,fp8,0,1.4692692756652832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,1,128,1,fp8,fp8,0,1.3453067143758137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,12,4,128,1,fp8,fp8,0,2.662442684173584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,2,128,1,float16,float16,0,1.9384907086690266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,2,128,1,float16,fp8,0,1.8775092760721843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,2,128,1,fp8,fp8,0,1.3502720197041829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,1,128,1,float16,float16,0,0.9901440143585205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,1,128,1,float16,fp8,0,0.9781386852264404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,4,128,1,float16,float16,0,1.9294293721516926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,4,128,1,fp8,fp8,0,1.4334880510965984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,12,4,128,1,float16,fp8,0,1.897066593170166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,12,128,1,float16,float16,0,1.0034613609313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,12,128,1,float16,fp8,0,0.9874826272328695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,12,128,1,fp8,fp8,0,0.7319093545277914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,1,128,1,fp8,fp8,0,0.7078080177307129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,2,128,1,float16,float16,0,0.9941333134969076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,2,128,1,float16,fp8,0,0.9846399625142416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,2,128,1,fp8,fp8,0,0.7275786399841309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,4,128,1,float16,float16,0,1.0020426909128826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,4,128,1,fp8,fp8,0,0.7211093107859293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,12,4,128,1,float16,fp8,0,0.9912266731262207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,1,128,1,float16,float16,0,0.5439146757125854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,12,128,1,float16,float16,0,0.5474239985148112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,1,128,1,float16,fp8,0,0.5443573395411173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,12,128,1,fp8,fp8,0,0.3792053461074829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,12,128,1,float16,fp8,0,0.5349066654841105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,1,128,1,fp8,fp8,0,0.36819199721018475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,2,128,1,float16,float16,0,0.5469653209050497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,2,128,1,float16,fp8,0,0.5420960187911987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,2,128,1,fp8,fp8,0,0.3685386578241984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,4,128,1,float16,float16,0,0.5521173477172852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,4,128,1,float16,fp8,0,0.5544693470001221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,12,128,1,float16,float16,0,0.27396267652511597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,12,4,128,1,fp8,fp8,0,0.3738186756769816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,12,128,1,float16,fp8,0,0.2712266643842061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,1,128,1,float16,fp8,0,0.2695840001106262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,12,128,1,fp8,fp8,0,0.21242666244506836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,1,128,1,float16,float16,0,0.27498666445414227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,1,128,1,fp8,fp8,0,0.20880534251530966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,2,128,1,float16,float16,0,0.27740800380706787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,2,128,1,fp8,fp8,0,0.208021342754364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,2,128,1,float16,fp8,0,0.2707253297170003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,4,128,1,float16,fp8,0,0.2720053394635518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,4,128,1,float16,float16,0,0.2776693304379781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,12,4,128,1,fp8,fp8,0,0.21130132675170898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,1,128,1,float16,fp8,0,3.699712117513021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,1,128,1,fp8,fp8,0,2.883530616760254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,1,128,1,float16,float16,0,3.7089974085489907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,2,128,1,float16,float16,0,3.779871940612793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,2,128,1,fp8,fp8,0,2.679109255472819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,2,128,1,float16,fp8,0,3.6268479029337564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,4,128,1,float16,fp8,0,3.7821226119995117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,4,128,1,float16,float16,0,3.831061363220215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,1,128,1,float16,float16,0,1.8674880663553874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,1,128,1,float16,fp8,0,1.8247839609781902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,12,128,1,float16,float16,0,2.137056032816569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,12,128,1,float16,fp8,0,1.8951412836710613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,12,4,128,1,fp8,fp8,0,2.7384745279947915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,1,128,1,fp8,fp8,0,1.354640007019043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,2,128,1,float16,float16,0,1.8719253540039062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,12,128,1,fp8,fp8,0,1.4202133814493816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,2,128,1,float16,fp8,0,1.8414559364318848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,2,128,1,fp8,fp8,0,1.3650186856587727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,1,128,1,float16,float16,0,0.9541973272959391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,1,128,1,float16,fp8,0,0.9379733403523763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,4,128,1,float16,float16,0,2.0381760597229004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,12,128,1,fp8,fp8,0,0.7379786968231201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,4,128,1,fp8,fp8,0,1.3901599248250325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,1,128,1,fp8,fp8,0,0.7000959714253744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,12,128,1,float16,float16,0,0.9848586718241373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,12,4,128,1,float16,fp8,0,1.861728032430013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,2,128,1,float16,float16,0,0.9572800000508627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,12,128,1,float16,fp8,0,0.9706719716389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,2,128,1,float16,fp8,0,0.9720746676127116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,2,128,1,fp8,fp8,0,0.7049012978871664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,4,128,1,float16,float16,0,0.9777386983235677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,4,128,1,float16,fp8,0,0.9527573585510254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,12,128,1,float16,float16,0,0.5224159955978394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,12,128,1,float16,fp8,0,0.5115626653035482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,1,128,1,float16,float16,0,0.5228799978892008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,12,4,128,1,fp8,fp8,0,0.7186559836069742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,12,128,1,fp8,fp8,0,0.3972640037536621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,1,128,1,float16,fp8,0,0.5023253361384074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,1,128,1,fp8,fp8,0,0.3617173433303833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,2,128,1,float16,float16,0,0.5136586825052897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,2,128,1,fp8,fp8,0,0.36489065488179523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,4,128,1,float16,float16,0,0.5401653448740641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,2,128,1,float16,fp8,0,0.5060319900512695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,4,128,1,float16,fp8,0,0.5109333197275797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,1,128,1,float16,float16,0,0.25781865914662677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,12,128,1,float16,fp8,0,0.2640373309453328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,12,128,1,fp8,fp8,0,0.21141332387924194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,12,4,128,1,fp8,fp8,0,0.37349867820739746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,1,128,1,float16,fp8,0,0.2559199929237366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,12,128,1,float16,float16,0,0.26628265778223675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,1,128,1,fp8,fp8,0,0.2015519936879476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,2,128,1,float16,fp8,0,0.25384000937143963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,2,128,1,fp8,fp8,0,0.20044799645741782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,2,128,1,float16,float16,0,0.25969066222508747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,4,128,1,float16,float16,0,0.26152533292770386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,4,128,1,fp8,fp8,0,0.21115734179814658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,12,4,128,1,float16,fp8,0,0.25757867097854614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,12,128,1,float16,float16,0,0.15287466843922934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,12,128,1,float16,fp8,0,0.1520960032939911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,12,128,1,fp8,fp8,0,0.1243893305460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,1,128,1,float16,float16,0,0.15083199739456177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,1,128,1,fp8,fp8,0,0.11543466647466023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,2,128,1,float16,float16,0,0.15251200397809347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,1,128,1,float16,fp8,0,0.14917332927385965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,4,128,1,float16,fp8,0,0.15041599671045938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,4,128,1,float16,float16,0,0.15334399541219076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,4,128,1,fp8,fp8,0,0.12084800004959106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,2,128,1,fp8,fp8,0,0.11776533722877502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,12,2,128,1,float16,fp8,0,0.1498240033785502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,1,128,1,float16,fp8,0,2.4434773127237954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,1,128,1,float16,float16,0,2.2990239461263022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,1,128,1,fp8,fp8,0,1.703839937845866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,2,128,1,float16,float16,0,2.3037333488464355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,2,128,1,float16,fp8,0,2.2598613103230796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,2,128,1,fp8,fp8,0,1.72107728322347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,4,128,1,float16,float16,0,2.341594696044922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,4,128,1,float16,fp8,0,2.288421312967936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,1,128,1,float16,fp8,0,1.1428799629211426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,1,128,1,float16,float16,0,1.158128023147583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,1,128,1,fp8,fp8,0,0.869658629099528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,12,128,1,float16,float16,0,1.2198506991068523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,12,4,128,1,fp8,fp8,0,1.9123093287150066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,12,128,1,float16,fp8,0,1.1923253536224365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,2,128,1,float16,float16,0,1.1653760274251301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,12,128,1,fp8,fp8,0,0.932032028834025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,2,128,1,fp8,fp8,0,0.8788800239562988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,2,128,1,float16,fp8,0,1.1505333582560222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,4,128,1,float16,float16,0,1.1786826451619465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,4,128,1,float16,fp8,0,1.1564586957295735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,1,128,1,float16,fp8,0,0.5919466813405355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,12,4,128,1,fp8,fp8,0,0.965562661488851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,1,128,1,float16,float16,0,0.6047626733779907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,12,128,1,float16,float16,0,0.6288640101750692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,12,128,1,float16,fp8,0,0.6180479923884074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,12,128,1,fp8,fp8,0,0.48839465777079266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,1,128,1,fp8,fp8,0,0.45536001523335773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,2,128,1,fp8,fp8,0,0.47789867719014484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,2,128,1,float16,float16,0,0.6058080196380615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,2,128,1,float16,fp8,0,0.5973706642786661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,4,128,1,float16,float16,0,0.6140053272247314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,4,128,1,float16,fp8,0,0.6011253197987875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,12,128,1,float16,float16,0,0.33872000376383465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,12,4,128,1,fp8,fp8,0,0.46931731700897217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,1,128,1,float16,fp8,0,0.3239733378092448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,12,128,1,fp8,fp8,0,0.2901866634686788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,12,128,1,float16,fp8,0,0.33416001001993817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,1,128,1,float16,float16,0,0.3285920023918152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,1,128,1,fp8,fp8,0,0.23939732710520426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,2,128,1,float16,fp8,0,0.32451732953389484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,2,128,1,fp8,fp8,0,0.24018667141596475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,2,128,1,float16,float16,0,0.3319680094718933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,4,128,1,float16,float16,0,0.33689598242441815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,1,128,1,float16,float16,0,0.1688106656074524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,4,128,1,float16,fp8,0,0.34253867467244464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,12,128,1,fp8,fp8,0,0.14492266376813254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,12,4,128,1,fp8,fp8,0,0.24453334013621011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,12,128,1,float16,fp8,0,0.17435733477274576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,1,128,1,float16,fp8,0,0.1660106678803762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,12,128,1,float16,float16,0,0.17592533429463705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,1,128,1,fp8,fp8,0,0.13369066516558328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,2,128,1,float16,fp8,0,0.16836265722910562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,2,128,1,float16,float16,0,0.16985066731770834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,2,128,1,fp8,fp8,0,0.13476266463597616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,4,128,1,float16,float16,0,0.17137600978215536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,4,128,1,float16,fp8,0,0.16850133736928305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,12,4,128,1,fp8,fp8,0,0.13885866602261862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,12,128,1,float16,fp8,0,0.10194133718808492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,1,128,1,float16,float16,0,0.10010133186976115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,12,128,1,fp8,fp8,0,0.08974933624267578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,1,128,1,float16,fp8,0,0.09803733229637146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,1,128,1,fp8,fp8,0,0.0794293334086736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,12,128,1,float16,float16,0,0.10270933310190837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,2,128,1,float16,float16,0,0.10008000334103902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,2,128,1,fp8,fp8,0,0.08021333316961925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,4,128,1,float16,float16,0,0.10106133421262105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,4,128,1,float16,fp8,0,0.10009599725405376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,4,128,1,fp8,fp8,0,0.08227199812730153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,12,2,128,1,float16,fp8,0,0.09868799646695454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,1,128,1,float16,float16,0,2.458288033803304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,2,128,1,float16,float16,0,2.506480058034261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,1,128,1,fp8,fp8,0,1.905509312947591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,2,128,1,fp8,fp8,0,2.1244212786356607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,1,128,1,float16,fp8,0,2.3355840047200522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,2,128,1,float16,fp8,0,2.346794605255127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,4,128,1,float16,float16,0,2.5197599728902182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,4,128,1,float16,fp8,0,2.3855466842651367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,1,128,1,float16,float16,0,1.2287306785583496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,1,128,1,float16,fp8,0,1.1737759908040364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,1,128,1,fp8,fp8,0,0.9528000354766846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,2,128,1,float16,float16,0,1.239909331003825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,12,128,1,float16,float16,0,1.3388373057047527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,12,128,1,fp8,fp8,0,1.197322686513265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,12,128,1,float16,fp8,0,1.2693226337432861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,2,128,1,float16,fp8,0,1.1795466740926106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,12,4,128,1,fp8,fp8,0,1.9622774124145508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,2,128,1,fp8,fp8,0,0.9628693262736002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,4,128,1,float16,float16,0,1.2560959657033284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,1,128,1,float16,float16,0,0.6186879873275757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,1,128,1,float16,fp8,0,0.6027733484903971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,12,128,1,float16,float16,0,0.7779040336608887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,1,128,1,fp8,fp8,0,0.4867946704228719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,4,128,1,fp8,fp8,0,0.9849332968393961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,12,4,128,1,float16,fp8,0,1.2014346917470295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,12,128,1,float16,fp8,0,0.6502346595128378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,12,128,1,fp8,fp8,0,0.5374933481216431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,2,128,1,float16,float16,0,0.6229120095570883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,2,128,1,fp8,fp8,0,0.5210239887237549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,2,128,1,float16,fp8,0,0.606117328008016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,4,128,1,float16,fp8,0,0.6177440087000529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,4,128,1,fp8,fp8,0,0.5056159893671671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,12,4,128,1,float16,float16,0,0.6325759887695312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,12,128,1,float16,float16,0,0.3539466857910156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,12,128,1,float16,fp8,0,0.3442986806233724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,1,128,1,float16,float16,0,0.3447466691335042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,1,128,1,float16,fp8,0,0.3216746648152669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,1,128,1,fp8,fp8,0,0.2521973252296448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,12,128,1,fp8,fp8,0,0.2933173378308614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,2,128,1,float16,fp8,0,0.3229653239250183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,2,128,1,float16,float16,0,0.33219732840855914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,2,128,1,fp8,fp8,0,0.256602664788564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,4,128,1,float16,float16,0,0.33618664741516113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,4,128,1,float16,fp8,0,0.34800533453623456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,12,128,1,float16,float16,0,0.18923733631769815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,12,4,128,1,fp8,fp8,0,0.2639946738878886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,12,128,1,float16,fp8,0,0.18457067012786865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,12,128,1,fp8,fp8,0,0.1551413337389628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,1,128,1,float16,float16,0,0.17097065846125284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,2,128,1,float16,float16,0,0.1779359976450602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,1,128,1,float16,fp8,0,0.16689600547154745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,1,128,1,fp8,fp8,0,0.13923733433087668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,2,128,1,float16,fp8,0,0.16833599408467612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,2,128,1,fp8,fp8,0,0.1399679978688558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,4,128,1,float16,fp8,0,0.17100266615549722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,4,128,1,float16,float16,0,0.17492800951004028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,12,128,1,float16,float16,0,0.11172800262769063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,1,128,1,float16,float16,0,0.09608532985051473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,12,4,128,1,fp8,fp8,0,0.14454399545987448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,1,128,1,float16,fp8,0,0.09431999921798706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,12,128,1,float16,fp8,0,0.10129066308339436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,1,128,1,fp8,fp8,0,0.07772799829641978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,12,128,1,fp8,fp8,0,0.08937600255012512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,2,128,1,float16,float16,0,0.09664533535639445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,2,128,1,float16,fp8,0,0.0965333382288615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,2,128,1,fp8,fp8,0,0.07946133116881053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,4,128,1,float16,fp8,0,0.09804800152778625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,12,128,1,float16,float16,0,0.06146133442719778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,4,128,1,float16,float16,0,0.09941866993904114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,1,128,1,float16,float16,0,0.060826669136683144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,12,128,1,fp8,fp8,0,0.05525333185990652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,12,4,128,1,fp8,fp8,0,0.08515733480453491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,1,128,1,float16,fp8,0,0.05991466840108236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,12,128,1,float16,fp8,0,0.06065600117047628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,1,128,1,fp8,fp8,0,0.051642666260401406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,2,128,1,float16,float16,0,0.06079466640949249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,2,128,1,fp8,fp8,0,0.05160533388455709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,4,128,1,float16,float16,0,0.06085866689682007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,4,128,1,fp8,fp8,0,0.052671998739242554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,2,128,1,float16,fp8,0,0.060362666845321655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,12,4,128,1,float16,fp8,0,0.06085866689682007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,1,128,1,float16,float16,0,1.569925308227539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,1,128,1,float16,fp8,0,1.5113120079040527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,1,128,1,fp8,fp8,0,1.257855971654256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,2,128,1,float16,float16,0,1.7830986976623535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,2,128,1,fp8,fp8,0,1.278490702311198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,2,128,1,float16,fp8,0,1.5217973391215007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,4,128,1,float16,fp8,0,1.5606719652811687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,4,128,1,float16,float16,0,1.626757303873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,1,128,1,float16,float16,0,0.7916639645894369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,1,128,1,float16,fp8,0,0.763375997543335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,12,128,1,float16,fp8,0,0.8394400278727213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,1,128,1,fp8,fp8,0,0.643392006556193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,12,128,1,float16,float16,0,0.864736000696818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,12,4,128,1,fp8,fp8,0,1.3147093454996746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,2,128,1,float16,float16,0,0.800597349802653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,2,128,1,float16,fp8,0,0.7734293142954508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,12,128,1,fp8,fp8,0,0.831274668375651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,2,128,1,fp8,fp8,0,0.6543733278910319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,1,128,1,float16,float16,0,0.43136000633239746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,1,128,1,float16,fp8,0,0.3959840138753255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,4,128,1,float16,float16,0,0.8135253588358561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,4,128,1,float16,fp8,0,0.7924160162607828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,12,4,128,1,fp8,fp8,0,0.6698079903920492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,12,128,1,fp8,fp8,0,0.3770666519800822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,12,128,1,float16,float16,0,0.4479946692784627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,1,128,1,fp8,fp8,0,0.35395201047261554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,12,128,1,float16,fp8,0,0.4367733399073283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,2,128,1,float16,float16,0,0.4124053319295247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,2,128,1,float16,fp8,0,0.4006933371225993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,4,128,1,float16,fp8,0,0.4111200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,12,128,1,float16,float16,0,0.28379199902216595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,4,128,1,float16,float16,0,0.4187093178431193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,2,128,1,fp8,fp8,0,0.3410826524098714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,12,4,128,1,fp8,fp8,0,0.34859732786814374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,1,128,1,float16,float16,0,0.22044267257054648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,1,128,1,float16,fp8,0,0.215503990650177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,1,128,1,fp8,fp8,0,0.18357332547505698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,12,128,1,float16,fp8,0,0.23431466023127237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,12,128,1,fp8,fp8,0,0.20250133673350015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,2,128,1,float16,float16,0,0.22270933787027994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,2,128,1,fp8,fp8,0,0.17698132991790771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,2,128,1,float16,fp8,0,0.21810134251912436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,4,128,1,float16,float16,0,0.22579199075698853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,1,128,1,float16,float16,0,0.11637333035469055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,4,128,1,float16,fp8,0,0.2226240038871765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,12,4,128,1,fp8,fp8,0,0.19398399194081625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,12,128,1,float16,float16,0,0.13049599528312683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,12,128,1,float16,fp8,0,0.1272533337275187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,12,128,1,fp8,fp8,0,0.11191999912261963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,1,128,1,float16,fp8,0,0.11524266997973125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,2,128,1,float16,float16,0,0.11738666892051697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,1,128,1,fp8,fp8,0,0.09937600294748943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,2,128,1,float16,fp8,0,0.11610666910807292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,2,128,1,fp8,fp8,0,0.10494933525721233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,4,128,1,float16,float16,0,0.12018666664759318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,4,128,1,float16,fp8,0,0.117658664782842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,12,4,128,1,fp8,fp8,0,0.10362666845321655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,12,128,1,float16,float16,0,0.07400533556938171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,1,128,1,float16,fp8,0,0.0674186646938324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,12,128,1,float16,fp8,0,0.07338133454322815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,12,128,1,fp8,fp8,0,0.06731200218200684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,1,128,1,fp8,fp8,0,0.057034666339556374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,2,128,1,float16,float16,0,0.06811733543872833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,2,128,1,float16,fp8,0,0.06771733363469441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,1,128,1,float16,float16,0,0.06868800024191539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,4,128,1,float16,float16,0,0.06986666719118755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,4,128,1,float16,fp8,0,0.06896000107129414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,2,128,1,fp8,fp8,0,0.05733866492907206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,12,4,128,1,fp8,fp8,0,0.06009600063165029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,12,128,1,float16,float16,0,0.04889066517353058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,12,128,1,float16,fp8,0,0.04862933357556661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,1,128,1,float16,fp8,0,0.048309331138928734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,12,128,1,fp8,fp8,0,0.04370133578777313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,1,128,1,float16,float16,0,0.04938133557637533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,1,128,1,fp8,fp8,0,0.04200533529122671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,2,128,1,float16,float16,0,0.04911999901135763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,2,128,1,float16,fp8,0,0.04836800197760264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,4,128,1,float16,fp8,0,0.04858666658401489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,4,128,1,float16,float16,0,0.04907733201980591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,4,128,1,fp8,fp8,0,0.04269866645336151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,12,2,128,1,fp8,fp8,0,0.04238933324813843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,1,128,1,float16,float16,0,1.462608019510905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,1,128,1,float16,fp8,0,1.459642728169759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,1,128,1,fp8,fp8,0,1.2245066960652669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,2,128,1,float16,fp8,0,1.6510772705078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,2,128,1,float16,float16,0,1.4868480364481609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,2,128,1,fp8,fp8,0,1.3258399963378906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,4,128,1,float16,float16,0,1.538090705871582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,4,128,1,float16,fp8,0,1.5547839800516765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,1,128,1,float16,float16,0,0.7478720347086588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,1,128,1,float16,fp8,0,0.746666669845581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,1,128,1,fp8,fp8,0,0.6169866720835367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,12,128,1,float16,float16,0,0.9414239724477133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,12,4,128,1,fp8,fp8,0,1.422378698984782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,2,128,1,float16,float16,0,0.7566186587015787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,12,128,1,float16,fp8,0,0.9105493227640787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,2,128,1,float16,fp8,0,0.753162701924642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,12,128,1,fp8,fp8,0,0.9559253056844076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,2,128,1,fp8,fp8,0,0.6659040053685507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,1,128,1,float16,float16,0,0.42156799634297687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,1,128,1,float16,fp8,0,0.38707200686136883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,4,128,1,float16,float16,0,0.7830773194630941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,4,128,1,float16,fp8,0,0.776688019434611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,12,4,128,1,fp8,fp8,0,0.7277599970499674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,12,128,1,float16,float16,0,0.477130651473999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,12,128,1,fp8,fp8,0,0.3998719851175944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,1,128,1,fp8,fp8,0,0.3349920113881429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,12,128,1,float16,fp8,0,0.4620213508605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,2,128,1,float16,fp8,0,0.38973867893218994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,2,128,1,float16,float16,0,0.39315199851989746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,2,128,1,fp8,fp8,0,0.338485320409139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,4,128,1,float16,float16,0,0.4065920114517212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,1,128,1,float16,float16,0,0.20729599396387735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,12,128,1,float16,float16,0,0.320362667242686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,4,128,1,float16,fp8,0,0.40221333503723145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,12,128,1,float16,fp8,0,0.24517865975697836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,12,4,128,1,fp8,fp8,0,0.3729493220647176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,1,128,1,fp8,fp8,0,0.16050133109092712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,1,128,1,float16,fp8,0,0.2063093384106954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,12,128,1,fp8,fp8,0,0.21426665782928467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,2,128,1,float16,float16,0,0.20925867557525635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,2,128,1,float16,fp8,0,0.23013333479563394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,2,128,1,fp8,fp8,0,0.16502400239308676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,4,128,1,float16,fp8,0,0.21517332394917807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,4,128,1,fp8,fp8,0,0.1876479983329773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,12,128,1,float16,float16,0,0.14648000399271646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,12,4,128,1,float16,float16,0,0.21850132942199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,12,128,1,float16,fp8,0,0.14316800236701965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,12,128,1,fp8,fp8,0,0.10852266351381938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,1,128,1,float16,float16,0,0.11443733175595601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,2,128,1,float16,float16,0,0.11231467127799988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,1,128,1,fp8,fp8,0,0.08839999636014302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,2,128,1,float16,fp8,0,0.11081066727638245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,1,128,1,float16,fp8,0,0.11032533645629883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,4,128,1,float16,float16,0,0.11755733688672383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,2,128,1,fp8,fp8,0,0.09180800120035808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,4,128,1,fp8,fp8,0,0.10544000069300334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,12,4,128,1,float16,fp8,0,0.11534933249155681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,12,128,1,float16,float16,0,0.07281066477298737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,12,128,1,float16,fp8,0,0.07076799869537354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,12,128,1,fp8,fp8,0,0.06169066826502482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,1,128,1,float16,fp8,0,0.06021333237489065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,2,128,1,float16,fp8,0,0.061893333991368614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,2,128,1,float16,float16,0,0.06193066636721293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,1,128,1,float16,float16,0,0.060085331400235496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,1,128,1,fp8,fp8,0,0.050240000089009605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,2,128,1,fp8,fp8,0,0.05263466636339823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,4,128,1,float16,float16,0,0.06417599817117055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,4,128,1,float16,fp8,0,0.06423999865849812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,12,4,128,1,fp8,fp8,0,0.05588266750176748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,12,128,1,float16,fp8,0,0.038933334251244865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,12,128,1,float16,float16,0,0.03922666609287262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,1,128,1,float16,float16,0,0.0359199990828832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,12,128,1,fp8,fp8,0,0.03695466617743174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,1,128,1,fp8,fp8,0,0.03141333411137263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,1,128,1,float16,fp8,0,0.03578133384386698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,2,128,1,float16,float16,0,0.036229332288106285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,2,128,1,float16,fp8,0,0.03646933287382126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,2,128,1,fp8,fp8,0,0.032357332607110344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,4,128,1,float16,float16,0,0.03714666763941447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,12,128,1,float16,float16,0,0.032458665470282234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,1,128,1,float16,float16,0,0.03182400017976761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,12,128,1,float16,fp8,0,0.03249066571394602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,4,128,1,float16,fp8,0,0.03736000011364619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,12,128,1,fp8,fp8,0,0.02864533414443334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,1,128,1,float16,fp8,0,0.032218667368094124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,1,128,1,fp8,fp8,0,0.027215999861558277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,12,4,128,1,fp8,fp8,0,0.03384000062942505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,2,128,1,float16,float16,0,0.03181333343187968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,2,128,1,float16,fp8,0,0.03206400076548258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,4,128,1,float16,fp8,0,0.03274133304754893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,4,128,1,float16,float16,0,0.032485333581765495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,2,128,1,fp8,fp8,0,0.027445333699385326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,12,4,128,1,fp8,fp8,0,0.02865600089232127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,1,128,1,float16,float16,0,1.1454400221506755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,1,128,1,float16,fp8,0,1.1329279740651448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,2,128,1,float16,fp8,0,1.167738676071167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,1,128,1,fp8,fp8,0,1.037295977274577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,2,128,1,fp8,fp8,0,1.1062133312225342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,2,128,1,float16,float16,0,1.1599786281585693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,4,128,1,float16,float16,0,1.267034689585368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,4,128,1,float16,fp8,0,1.4880107243855794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,1,128,1,float16,float16,0,0.5820639928181967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,1,128,1,float16,fp8,0,0.5776693423589071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,1,128,1,fp8,fp8,0,0.5116480191548666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,12,128,1,float16,float16,0,0.7928746541341146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,12,4,128,1,fp8,fp8,0,1.2229599952697754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,12,128,1,float16,fp8,0,0.7564106782277426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,2,128,1,float16,float16,0,0.6940373579661051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,12,128,1,fp8,fp8,0,0.6738506952921549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,2,128,1,fp8,fp8,0,0.5671146710713705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,2,128,1,float16,fp8,0,0.5916693210601807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,1,128,1,float16,float16,0,0.30275734265645343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,4,128,1,float16,float16,0,0.6272426843643188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,1,128,1,float16,fp8,0,0.30084266265233356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,12,128,1,float16,float16,0,0.4039520025253296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,4,128,1,fp8,fp8,0,0.618282675743103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,12,4,128,1,float16,fp8,0,0.6191360155741373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,12,128,1,fp8,fp8,0,0.3447519938151042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,1,128,1,fp8,fp8,0,0.26654932896296185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,2,128,1,float16,float16,0,0.30844799677530926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,12,128,1,float16,fp8,0,0.5442560116449991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,2,128,1,float16,fp8,0,0.3052106698354085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,2,128,1,fp8,fp8,0,0.2879413366317749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,4,128,1,float16,float16,0,0.3249866763750712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,12,128,1,float16,float16,0,0.21276267369588217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,4,128,1,fp8,fp8,0,0.31679999828338623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,12,128,1,float16,fp8,0,0.20296533902486166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,1,128,1,float16,fp8,0,0.1618773341178894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,1,128,1,float16,float16,0,0.1634986698627472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,1,128,1,fp8,fp8,0,0.1348960002263387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,12,4,128,1,float16,fp8,0,0.31915199756622314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,12,128,1,fp8,fp8,0,0.26337599754333496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,2,128,1,float16,float16,0,0.16595733165740967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,2,128,1,fp8,fp8,0,0.14078399538993835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,2,128,1,float16,fp8,0,0.16390933593114218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,4,128,1,float16,float16,0,0.17468265692392984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,4,128,1,fp8,fp8,0,0.1592586636543274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,12,4,128,1,float16,fp8,0,0.17245866854985556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,1,128,1,float16,float16,0,0.09035199880599976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,12,128,1,float16,float16,0,0.12961066762606302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,1,128,1,float16,fp8,0,0.09038933118184407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,12,128,1,fp8,fp8,0,0.09644266963005066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,2,128,1,float16,float16,0,0.09218133489290874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,1,128,1,fp8,fp8,0,0.07551466425259908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,12,128,1,float16,fp8,0,0.1792853275934855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,2,128,1,fp8,fp8,0,0.07729599873224895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,2,128,1,float16,fp8,0,0.09130133191744487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,4,128,1,float16,fp8,0,0.09609066446622212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,12,128,1,float16,float16,0,0.0634080022573471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,4,128,1,float16,float16,0,0.09755733609199524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,12,4,128,1,fp8,fp8,0,0.08477866649627686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,12,128,1,float16,fp8,0,0.06099733213583628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,1,128,1,float16,float16,0,0.050661335388819374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,12,128,1,fp8,fp8,0,0.06833066542943318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,1,128,1,fp8,fp8,0,0.04330666859944662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,1,128,1,float16,fp8,0,0.050442665815353394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,2,128,1,float16,float16,0,0.051594664653142296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,2,128,1,fp8,fp8,0,0.04498666524887085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,4,128,1,float16,float16,0,0.05499733487764994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,4,128,1,float16,fp8,0,0.05429866909980774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,12,128,1,float16,float16,0,0.03499733408292135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,4,128,1,fp8,fp8,0,0.05268799761931101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,12,2,128,1,float16,fp8,0,0.05182399849096934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,12,128,1,fp8,fp8,0,0.03345066557327906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,12,128,1,float16,fp8,0,0.03562133262554804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,1,128,1,float16,float16,0,0.0316746657093366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,1,128,1,float16,fp8,0,0.03123733401298523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,2,128,1,float16,float16,0,0.03166933357715607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,1,128,1,fp8,fp8,0,0.029130667448043823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,2,128,1,float16,fp8,0,0.03269333392381668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,2,128,1,fp8,fp8,0,0.030250666042168934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,4,128,1,float16,float16,0,0.03334933271010717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,4,128,1,fp8,fp8,0,0.03170666595300039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,12,4,128,1,float16,fp8,0,0.033215999603271484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,12,128,1,float16,float16,0,0.026191999514897663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,12,128,1,float16,fp8,0,0.026474667092164356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,12,128,1,fp8,fp8,0,0.024480000138282776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,1,128,1,float16,float16,0,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,1,128,1,fp8,fp8,0,0.02288000037272771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,2,128,1,float16,float16,0,0.025663999219735462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,2,128,1,fp8,fp8,0,0.023200000325838726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,2,128,1,float16,fp8,0,0.025765334566434223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,4,128,1,float16,float16,0,0.02607999990383784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,4,128,1,fp8,fp8,0,0.024405332903067272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,4,128,1,float16,fp8,0,0.02659733345111211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,12,1,128,1,float16,fp8,0,0.025301332275072735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,12,128,1,float16,float16,0,0.02253866692384084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,12,128,1,float16,fp8,0,0.022592000663280487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,1,128,1,float16,fp8,0,0.02207999924818675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,1,128,1,float16,float16,0,0.02203733225663503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,12,128,1,fp8,fp8,0,0.020181333025296528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,1,128,1,fp8,fp8,0,0.019695999721686046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,2,128,1,float16,float16,0,0.02195200075705846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,2,128,1,fp8,fp8,0,0.019738666713237762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,4,128,1,float16,float16,0,0.022522665560245514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,2,128,1,float16,fp8,0,0.022458667556444805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,4,128,1,fp8,fp8,0,0.020143999407688778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,12,4,128,1,float16,fp8,0,0.02239999920129776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,1,128,1,float16,fp8,0,0.5883946816126505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,1,128,1,float16,float16,0,0.4984159866968791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,1,128,1,fp8,fp8,0,0.4248640139897664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,2,128,1,float16,float16,0,0.5114719867706299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,2,128,1,float16,fp8,0,0.5070879856745402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,2,128,1,fp8,fp8,0,0.4907519817352295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,4,128,1,float16,fp8,0,0.5357226530710856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,4,128,1,float16,float16,0,0.549397349357605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,1,128,1,float16,float16,0,0.2589706579844157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,1,128,1,fp8,fp8,0,0.22154666980107626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,12,128,1,float16,fp8,0,0.35468800862630206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,12,128,1,float16,float16,0,0.370469331741333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,1,128,1,float16,fp8,0,0.2582559982935588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,2,128,1,float16,float16,0,0.26679466168085736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,12,4,128,1,fp8,fp8,0,0.6449919939041138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,12,128,1,fp8,fp8,0,0.3068160017331441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,2,128,1,float16,fp8,0,0.2637333273887634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,2,128,1,fp8,fp8,0,0.24970134099324545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,4,128,1,float16,float16,0,0.28617600599924725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,4,128,1,float16,fp8,0,0.27750933170318604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,12,128,1,float16,fp8,0,0.28178133567174274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,12,128,1,float16,float16,0,0.19587733348210654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,12,4,128,1,fp8,fp8,0,0.28571200370788574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,1,128,1,float16,float16,0,0.13944000005722046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,12,128,1,fp8,fp8,0,0.1683786710103353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,1,128,1,float16,fp8,0,0.1393226683139801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,1,128,1,fp8,fp8,0,0.11875200271606445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,2,128,1,float16,fp8,0,0.17203734318415323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,2,128,1,fp8,fp8,0,0.1260479986667633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,2,128,1,float16,float16,0,0.1436746617158254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,4,128,1,fp8,fp8,0,0.14622933665911356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,4,128,1,float16,fp8,0,0.15101333459218344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,12,128,1,float16,fp8,0,0.11192533373832703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,12,4,128,1,float16,float16,0,0.15541866421699524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,12,128,1,float16,float16,0,0.11772800485293071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,1,128,1,float16,float16,0,0.08012799918651581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,12,128,1,fp8,fp8,0,0.1213759978612264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,1,128,1,float16,fp8,0,0.08046933511892955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,1,128,1,fp8,fp8,0,0.06720000008742015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,2,128,1,float16,float16,0,0.08243200182914734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,2,128,1,float16,fp8,0,0.08181333541870117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,4,128,1,float16,float16,0,0.08889066179593404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,2,128,1,fp8,fp8,0,0.07150933146476746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,4,128,1,float16,fp8,0,0.08796800176302592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,12,4,128,1,fp8,fp8,0,0.08634666601816814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,1,128,1,float16,float16,0,0.04574400186538696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,12,128,1,float16,fp8,0,0.056847999493281044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,12,128,1,fp8,fp8,0,0.05180799961090088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,1,128,1,float16,fp8,0,0.04570133487383524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,12,128,1,float16,float16,0,0.06046933432420095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,2,128,1,float16,fp8,0,0.04702933132648468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,2,128,1,float16,float16,0,0.0473333348830541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,1,128,1,fp8,fp8,0,0.03898133337497711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,2,128,1,fp8,fp8,0,0.04119999955097834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,4,128,1,float16,float16,0,0.049813335140546165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,4,128,1,float16,fp8,0,0.04996799925963084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,12,4,128,1,fp8,fp8,0,0.04523199796676636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,1,128,1,float16,fp8,0,0.030495998760064442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,12,128,1,float16,fp8,0,0.033530667424201965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,1,128,1,float16,float16,0,0.03053866575161616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,12,128,1,float16,float16,0,0.03370666752258936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,1,128,1,fp8,fp8,0,0.027482666075229645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,2,128,1,float16,float16,0,0.031514666974544525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,12,128,1,fp8,fp8,0,0.031504000226656594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,2,128,1,float16,fp8,0,0.031162666777769726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,2,128,1,fp8,fp8,0,0.027903998891512554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,4,128,1,float16,float16,0,0.03252800057331721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,4,128,1,float16,fp8,0,0.03268266717592875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,12,4,128,1,fp8,fp8,0,0.02956799914439519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,12,128,1,float16,fp8,0,0.023978665471076965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,12,128,1,float16,float16,0,0.023797333240509033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,1,128,1,float16,fp8,0,0.023007998863856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,1,128,1,float16,float16,0,0.02272533377011617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,1,128,1,fp8,fp8,0,0.021040000021457672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,12,128,1,fp8,fp8,0,0.023221333821614582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,2,128,1,float16,float16,0,0.02313599983851115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,2,128,1,fp8,fp8,0,0.021317332983016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,2,128,1,float16,fp8,0,0.0232640008131663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,4,128,1,float16,float16,0,0.02369066576162974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,4,128,1,fp8,fp8,0,0.02260799954334895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,12,128,1,float16,float16,0,0.019685332973798115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,12,4,128,1,float16,fp8,0,0.024127999941507976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,12,128,1,float16,fp8,0,0.019866666446129482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,12,128,1,fp8,fp8,0,0.01851733277241389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,1,128,1,float16,float16,0,0.018853332847356796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,1,128,1,fp8,fp8,0,0.01766933376590411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,1,128,1,float16,fp8,0,0.018976000448067982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,2,128,1,float16,float16,0,0.019007999449968338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,2,128,1,float16,fp8,0,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,2,128,1,fp8,fp8,0,0.017637333522240322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,4,128,1,float16,float16,0,0.019173332800467808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,4,128,1,fp8,fp8,0,0.017946666727463405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,12,128,1,float16,float16,0,0.017583999782800674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,12,128,1,float16,fp8,0,0.018922666708628338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,12,128,1,fp8,fp8,0,0.017237332959969837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,12,4,128,1,float16,fp8,0,0.019365333020687103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,1,128,1,float16,fp8,0,0.018186666071414948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,1,128,1,float16,float16,0,0.0180479995906353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,1,128,1,fp8,fp8,0,0.016976000120242436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,2,128,1,float16,float16,0,0.01826133330663045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,2,128,1,float16,fp8,0,0.018272000054518383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,2,128,1,fp8,fp8,0,0.017045332739750545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,4,128,1,float16,float16,0,0.01809599995613098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,4,128,1,fp8,fp8,0,0.0174346665541331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,12,4,128,1,float16,fp8,0,0.01854933301607768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,1,128,1,float16,float16,0,0.3017333348592122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,1,128,1,float16,fp8,0,0.24681599934895834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,1,128,1,fp8,fp8,0,0.2269173264503479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,2,128,1,float16,fp8,0,0.2512213389078776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,2,128,1,fp8,fp8,0,0.2520586649576823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,2,128,1,float16,float16,0,0.25382399559020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,4,128,1,float16,float16,0,0.27056533098220825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,4,128,1,float16,fp8,0,0.2663946747779846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,1,128,1,float16,float16,0,0.13523733615875244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,1,128,1,float16,fp8,0,0.13552000125249228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,12,4,128,1,fp8,fp8,0,0.31199999650319415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,1,128,1,fp8,fp8,0,0.11880532900492351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,12,128,1,float16,fp8,0,0.1967680056889852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,2,128,1,float16,float16,0,0.13875733812650046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,12,128,1,fp8,fp8,0,0.16009599963823953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,12,128,1,float16,float16,0,0.204858660697937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,2,128,1,float16,fp8,0,0.13809067010879517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,4,128,1,float16,float16,0,0.1957119901974996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,2,128,1,fp8,fp8,0,0.12648533781369528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,4,128,1,float16,fp8,0,0.14667733510335287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,12,128,1,float16,float16,0,0.10889599720637004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,12,128,1,float16,fp8,0,0.10552000006039937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,1,128,1,float16,float16,0,0.07638933261235555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,12,4,128,1,fp8,fp8,0,0.1442080040772756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,12,128,1,fp8,fp8,0,0.08865066369374593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,1,128,1,float16,fp8,0,0.08131733536720276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,1,128,1,fp8,fp8,0,0.06734399994214375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,2,128,1,float16,fp8,0,0.07845333218574524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,2,128,1,float16,float16,0,0.07813866436481476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,2,128,1,fp8,fp8,0,0.07111999889214833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,4,128,1,float16,float16,0,0.08438400427500407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,4,128,1,float16,fp8,0,0.08275199929873149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,12,4,128,1,fp8,fp8,0,0.07653333246707916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,12,128,1,float16,float16,0,0.07533333202203114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,12,128,1,float16,fp8,0,0.05669866502285004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,1,128,1,float16,float16,0,0.04331733286380768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,1,128,1,float16,fp8,0,0.04450133442878723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,12,128,1,fp8,fp8,0,0.05171200136343638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,1,128,1,fp8,fp8,0,0.03866666555404663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,2,128,1,float16,float16,0,0.04422933359940847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,2,128,1,fp8,fp8,0,0.04091733445723852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,2,128,1,float16,fp8,0,0.04456000030040741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,4,128,1,float16,fp8,0,0.04738133152325948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,4,128,1,fp8,fp8,0,0.045370668172836304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,12,4,128,1,float16,float16,0,0.047242666284243263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,12,128,1,float16,float16,0,0.03404266635576884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,12,128,1,float16,fp8,0,0.03348266581694285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,1,128,1,float16,float16,0,0.030421334008375805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,12,128,1,fp8,fp8,0,0.030602666238943737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,1,128,1,fp8,fp8,0,0.027189334233601887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,1,128,1,float16,fp8,0,0.030394665896892548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,2,128,1,float16,float16,0,0.030640001098314922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,2,128,1,float16,fp8,0,0.030320001145203907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,4,128,1,float16,float16,0,0.032186667124430336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,2,128,1,fp8,fp8,0,0.02779199928045273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,4,128,1,float16,fp8,0,0.03215466688076655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,12,4,128,1,fp8,fp8,0,0.029391999046007793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,12,128,1,float16,float16,0,0.023183998962243397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,12,128,1,float16,fp8,0,0.02348266790310542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,12,128,1,fp8,fp8,0,0.022645334402720135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,1,128,1,float16,fp8,0,0.021589333812395733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,1,128,1,float16,float16,0,0.021509334444999695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,2,128,1,float16,fp8,0,0.02187199890613556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,2,128,1,float16,float16,0,0.02186133215824763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,2,128,1,fp8,fp8,0,0.021162666380405426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,1,128,1,fp8,fp8,0,0.021002667645613354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,4,128,1,float16,float16,0,0.022682666778564453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,4,128,1,float16,fp8,0,0.022810667753219604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,12,4,128,1,fp8,fp8,0,0.022384000321229298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,12,128,1,float16,float16,0,0.01766933376590411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,12,128,1,float16,fp8,0,0.018794666975736618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,12,128,1,fp8,fp8,0,0.01830400029818217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,1,128,1,float16,float16,0,0.01730666682124138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,1,128,1,fp8,fp8,0,0.017530667285124462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,2,128,1,float16,float16,0,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,1,128,1,float16,fp8,0,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,2,128,1,float16,fp8,0,0.01785600061217944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,2,128,1,fp8,fp8,0,0.017632000148296356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,4,128,1,float16,float16,0,0.017968000223239262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,4,128,1,float16,fp8,0,0.018133333573738735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,12,4,128,1,fp8,fp8,0,0.017797333498795826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,12,128,1,float16,fp8,0,0.017194667210181553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,12,128,1,float16,float16,0,0.017018667111794155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,12,128,1,fp8,fp8,0,0.017338667064905167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,1,128,1,float16,float16,0,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,1,128,1,float16,fp8,0,0.01709866647919019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,1,128,1,fp8,fp8,0,0.01716800034046173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,2,128,1,float16,float16,0,0.016927999754746754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,2,128,1,float16,fp8,0,0.01708799973130226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,2,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,4,128,1,float16,fp8,0,0.017477333545684814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,4,128,1,float16,float16,0,0.017184000462293625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,12,4,128,1,fp8,fp8,0,0.017071999609470367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,12,128,1,float16,float16,0,0.01579733317097028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,12,128,1,float16,fp8,0,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,12,128,1,fp8,fp8,0,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,1,128,1,float16,float16,0,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,1,128,1,float16,fp8,0,0.01676799977819125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,1,128,1,fp8,fp8,0,0.01681600014368693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,2,128,1,float16,fp8,0,0.016965333372354507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,2,128,1,float16,float16,0,0.016469333320856094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,2,128,1,fp8,fp8,0,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,4,128,1,float16,float16,0,0.01661866654952367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,4,128,1,float16,fp8,0,0.016714667280515034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,12,4,128,1,fp8,fp8,0,0.01666133354107539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,1,128,1,float16,float16,0,0.15595199664433798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,1,128,1,fp8,fp8,0,0.15986667076746622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,2,128,1,fp8,fp8,0,0.16576000054677328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,2,128,1,float16,fp8,0,0.1606933375199636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,2,128,1,float16,float16,0,0.15939733386039734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,4,128,1,float16,float16,0,0.21477866172790527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,1,128,1,float16,fp8,0,0.15707199772198996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,4,128,1,float16,fp8,0,0.16845866044362387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,1,128,1,float16,fp8,0,0.08759466807047527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,12,128,1,float16,float16,0,0.10750933488210042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,2,128,1,float16,float16,0,0.08680533369382222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,1,128,1,float16,float16,0,0.08501332998275757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,12,128,1,float16,fp8,0,0.10367467006047566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,12,128,1,fp8,fp8,0,0.10708266496658325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,1,128,1,fp8,fp8,0,0.08990933497746785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,12,4,128,1,fp8,fp8,0,0.18313600619633993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,2,128,1,float16,fp8,0,0.09278399745623271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,2,128,1,fp8,fp8,0,0.09316266576449077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,4,128,1,float16,float16,0,0.09289066990216573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,4,128,1,fp8,fp8,0,0.09751466910044353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,12,128,1,float16,float16,0,0.05857066810131073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,12,128,1,float16,fp8,0,0.05723733206590017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,12,128,1,fp8,fp8,0,0.06177066763242086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,1,128,1,float16,float16,0,0.04736533264319102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,12,4,128,1,float16,fp8,0,0.0941546658674876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,1,128,1,float16,fp8,0,0.04833066463470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,1,128,1,fp8,fp8,0,0.049551998575528465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,2,128,1,float16,float16,0,0.0487360010544459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,2,128,1,float16,fp8,0,0.04878933231035868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,2,128,1,fp8,fp8,0,0.05136000116666158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,4,128,1,float16,float16,0,0.05136000116666158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,4,128,1,float16,fp8,0,0.05226133267084757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,12,128,1,float16,fp8,0,0.03053866575161616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,12,128,1,float16,float16,0,0.03137599925200144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,1,128,1,float16,float16,0,0.029845332105954487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,12,128,1,fp8,fp8,0,0.034143999218940735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,1,128,1,float16,fp8,0,0.029701332251230877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,1,128,1,fp8,fp8,0,0.030581332743167877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,12,4,128,1,fp8,fp8,0,0.05507733424504598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,2,128,1,float16,float16,0,0.03010133405526479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,2,128,1,float16,fp8,0,0.030005333324273426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,2,128,1,fp8,fp8,0,0.03151999910672506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,4,128,1,float16,float16,0,0.031317333380381264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,4,128,1,float16,fp8,0,0.03129599988460541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,12,4,128,1,fp8,fp8,0,0.03339199970165888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,12,128,1,float16,float16,0,0.024485332270463307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,12,128,1,fp8,fp8,0,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,1,128,1,float16,float16,0,0.023647998770078022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,12,128,1,float16,fp8,0,0.024720000723997753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,1,128,1,float16,fp8,0,0.023658665517965954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,1,128,1,fp8,fp8,0,0.024282666544119518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,2,128,1,float16,float16,0,0.024005333582560223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,2,128,1,float16,fp8,0,0.023989332218964893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,2,128,1,fp8,fp8,0,0.0245919997493426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,4,128,1,float16,float16,0,0.024405332903067272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,4,128,1,float16,fp8,0,0.024383999407291412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,12,128,1,float16,float16,0,0.017583999782800674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,12,128,1,float16,fp8,0,0.01749333366751671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,12,128,1,fp8,fp8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,1,128,1,float16,float16,0,0.016741332908471424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,1,128,1,float16,fp8,0,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,1,128,1,fp8,fp8,0,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,12,4,128,1,fp8,fp8,0,0.025807999074459076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,2,128,1,float16,fp8,0,0.016762666404247284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,2,128,1,float16,float16,0,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,2,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,4,128,1,float16,float16,0,0.01716800034046173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,12,128,1,float16,float16,0,0.015840000162522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,4,128,1,fp8,fp8,0,0.017621333400408428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,12,4,128,1,float16,fp8,0,0.016976000120242436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,12,128,1,fp8,fp8,0,0.017050666113694508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,1,128,1,float16,float16,0,0.015583999454975128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,12,128,1,float16,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,1,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,1,128,1,float16,fp8,0,0.01618133361140887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,2,128,1,float16,float16,0,0.015781333049138386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,2,128,1,float16,fp8,0,0.016085332880417507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,2,128,1,fp8,fp8,0,0.016938666502634685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,4,128,1,float16,fp8,0,0.016048000504573185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,4,128,1,float16,float16,0,0.015589332828919092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,12,128,1,float16,float16,0,0.015189333508412043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,12,128,1,float16,fp8,0,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,1,128,1,float16,fp8,0,0.01590399940808614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,12,128,1,fp8,fp8,0,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,1,128,1,float16,float16,0,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,12,4,128,1,fp8,fp8,0,0.017370666066805523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,1,128,1,fp8,fp8,0,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,2,128,1,float16,float16,0,0.015200000256299973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,2,128,1,float16,fp8,0,0.015717333803574245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,2,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,4,128,1,float16,float16,0,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,4,128,1,fp8,fp8,0,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,12,4,128,1,float16,fp8,0,0.015824000040690105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,12,128,1,float16,fp8,0,0.014885333677132925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,12,128,1,fp8,fp8,0,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,1,128,1,float16,float16,0,0.015040000279744467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,12,128,1,float16,float16,0,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,1,128,1,float16,fp8,0,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,2,128,1,float16,float16,0,0.014938666174809137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,1,128,1,fp8,fp8,0,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,2,128,1,float16,fp8,0,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,2,128,1,fp8,fp8,0,0.016016000260909397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,4,128,1,float16,float16,0,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,4,128,1,float16,fp8,0,0.015594666202863058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,1,128,1,float16,float16,0,0.10669333736101787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,12,4,128,1,fp8,fp8,0,0.016303999970356624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,1,128,1,float16,fp8,0,0.10692266623179118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,2,128,1,float16,float16,0,0.11727466185887654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,1,128,1,fp8,fp8,0,0.13147733608881632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,2,128,1,float16,fp8,0,0.10956799983978271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,2,128,1,fp8,fp8,0,0.1353600025177002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,4,128,1,float16,float16,0,0.11820266644159953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,12,128,1,float16,float16,0,0.07002666592597961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,4,128,1,float16,fp8,0,0.11540800333023071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,1,128,1,float16,float16,0,0.05793599784374237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,12,128,1,float16,fp8,0,0.08051200211048126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,12,128,1,fp8,fp8,0,0.082805335521698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,1,128,1,float16,fp8,0,0.0584746648867925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,12,4,128,1,fp8,fp8,0,0.1384053329626719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,1,128,1,fp8,fp8,0,0.07037333150704701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,2,128,1,float16,float16,0,0.05896000067392985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,2,128,1,fp8,fp8,0,0.07293866574764252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,4,128,1,float16,float16,0,0.06439466774463654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,2,128,1,float16,fp8,0,0.05906666815280914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,4,128,1,fp8,fp8,0,0.07709866762161255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,12,128,1,float16,fp8,0,0.035930665830771126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,12,4,128,1,float16,fp8,0,0.06259733438491821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,12,128,1,float16,float16,0,0.03689600030581156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,12,128,1,fp8,fp8,0,0.044549331068992615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,1,128,1,fp8,fp8,0,0.04104000081618627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,1,128,1,float16,fp8,0,0.03513066718975703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,1,128,1,float16,float16,0,0.03461333364248276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,2,128,1,float16,float16,0,0.03506666670242945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,2,128,1,float16,fp8,0,0.03526400029659271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,4,128,1,float16,fp8,0,0.03613866617282232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,12,128,1,float16,fp8,0,0.025173333783944447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,4,128,1,fp8,fp8,0,0.043391997615496315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,2,128,1,fp8,fp8,0,0.04234666625658671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,12,128,1,fp8,fp8,0,0.029546665648619335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,1,128,1,float16,float16,0,0.024325333535671234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,12,128,1,float16,float16,0,0.024879999458789825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,12,4,128,1,float16,float16,0,0.03606933355331421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,1,128,1,float16,fp8,0,0.024527999262015026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,1,128,1,fp8,fp8,0,0.027776000400384266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,2,128,1,float16,float16,0,0.024666666984558105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,2,128,1,fp8,fp8,0,0.02826133370399475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,2,128,1,float16,fp8,0,0.025055999557177227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,12,128,1,float16,fp8,0,0.019109333554903667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,4,128,1,float16,float16,0,0.025226667523384094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,12,128,1,fp8,fp8,0,0.021295999487241108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,1,128,1,fp8,fp8,0,0.02088533341884613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,1,128,1,float16,float16,0,0.018383999665578205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,4,128,1,fp8,fp8,0,0.029525332152843475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,12,4,128,1,float16,fp8,0,0.02510933329661687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,12,128,1,float16,float16,0,0.0191040001809597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,2,128,1,float16,float16,0,0.018373332917690277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,2,128,1,fp8,fp8,0,0.02089066555102666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,1,128,1,float16,fp8,0,0.01877333347996076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,2,128,1,float16,fp8,0,0.01855466639002164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,4,128,1,float16,float16,0,0.01887999971707662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,12,128,1,fp8,fp8,0,0.017445333302021027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,12,128,1,float16,float16,0,0.015504000087579092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,4,128,1,fp8,fp8,0,0.02130666623512904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,12,128,1,float16,fp8,0,0.01570133368174235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,12,4,128,1,float16,fp8,0,0.018837332725524902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,1,128,1,float16,float16,0,0.014954666296641031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,1,128,1,float16,fp8,0,0.01543466622630755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,1,128,1,fp8,fp8,0,0.016832000265518825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,2,128,1,float16,float16,0,0.015103999525308609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,2,128,1,fp8,fp8,0,0.01721599946419398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,2,128,1,float16,fp8,0,0.015504000087579092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,4,128,1,float16,float16,0,0.015247999380032221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,12,128,1,float16,float16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,12,128,1,float16,fp8,0,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,12,128,1,fp8,fp8,0,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,4,128,1,fp8,fp8,0,0.017312000195185345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,1,128,1,float16,fp8,0,0.015072000523408255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,1,128,1,float16,float16,0,0.014554666976133982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,1,128,1,fp8,fp8,0,0.016656000167131424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,2,128,1,float16,float16,0,0.014479999740918478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,12,4,128,1,float16,fp8,0,0.015439999600251516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,2,128,1,fp8,fp8,0,0.016261332978804905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,2,128,1,float16,fp8,0,0.015066667149464289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,4,128,1,float16,float16,0,0.014549333602190018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,4,128,1,float16,fp8,0,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,12,128,1,float16,float16,0,0.014309333016475042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,12,128,1,float16,fp8,0,0.014197333405415217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,1,128,1,float16,float16,0,0.014149333039919535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,12,4,128,1,fp8,fp8,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,1,128,1,float16,fp8,0,0.014826666563749313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,12,128,1,fp8,fp8,0,0.016250666230916977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,1,128,1,fp8,fp8,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,2,128,1,float16,float16,0,0.014240000396966934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,2,128,1,fp8,fp8,0,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,2,128,1,float16,fp8,0,0.014912000546852747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,4,128,1,float16,fp8,0,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,4,128,1,float16,float16,0,0.01414399966597557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,12,4,128,1,fp8,fp8,0,0.01617066686352094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,12,128,1,fp8,fp8,0,0.015754666179418564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,1,128,1,float16,float16,0,0.014005333185195923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,1,128,1,float16,fp8,0,0.014741333822409311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,1,128,1,fp8,fp8,0,0.01611199975013733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,2,128,1,float16,float16,0,0.013989333063364029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,12,128,1,float16,float16,0,0.014042666802803675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,12,128,1,float16,fp8,0,0.014426667243242264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,2,128,1,float16,fp8,0,0.014773332824309668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,2,128,1,fp8,fp8,0,0.015989333391189575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,4,128,1,float16,float16,0,0.014085333794355392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,4,128,1,float16,fp8,0,0.014746667196353277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,12,4,128,1,fp8,fp8,0,0.016117333124081295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,1,128,1,float16,float16,0,0.09131200114885966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,2,128,1,float16,float16,0,0.09212799866994222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,1,128,1,fp8,fp8,0,0.11150933305422465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,2,128,1,float16,fp8,0,0.09257599711418152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,1,128,1,float16,fp8,0,0.0918293297290802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,2,128,1,fp8,fp8,0,0.11311999956766765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,4,128,1,float16,float16,0,0.0955573320388794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,4,128,1,float16,fp8,0,0.09619733691215515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,12,4,128,1,fp8,fp8,0,0.11569066842397054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,12,128,1,float16,float16,0,0.04969066878159841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,12,128,1,fp8,fp8,0,0.0647573322057724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,1,128,1,float16,fp8,0,0.05229333539803823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,12,128,1,float16,fp8,0,0.049141332507133484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,2,128,1,float16,float16,0,0.052576000491778054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,1,128,1,fp8,fp8,0,0.06228800117969513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,1,128,1,float16,float16,0,0.052058666944503784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,2,128,1,float16,fp8,0,0.05227733155091604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,2,128,1,fp8,fp8,0,0.06294933458169301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,4,128,1,float16,float16,0,0.05317866802215576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,12,128,1,float16,float16,0,0.03188266605138779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,12,128,1,float16,fp8,0,0.03182400017976761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,1,128,1,float16,float16,0,0.03234133372704188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,4,128,1,float16,fp8,0,0.05323199927806854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,12,4,128,1,fp8,fp8,0,0.06471999982992808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,12,128,1,fp8,fp8,0,0.04011200120051702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,1,128,1,float16,fp8,0,0.0325546662012736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,1,128,1,fp8,fp8,0,0.03864533454179764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,2,128,1,float16,float16,0,0.03298133363326391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,2,128,1,float16,fp8,0,0.03305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,2,128,1,fp8,fp8,0,0.0388373335202535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,4,128,1,float16,float16,0,0.03349866718053818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,4,128,1,float16,fp8,0,0.03387733300526937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,12,4,128,1,fp8,fp8,0,0.04001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,12,128,1,float16,fp8,0,0.021695998807748158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,12,128,1,fp8,fp8,0,0.025301332275072735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,1,128,1,float16,fp8,0,0.021477334201335907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,12,128,1,float16,float16,0,0.021456000705560047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,1,128,1,float16,float16,0,0.02128000060717265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,1,128,1,fp8,fp8,0,0.024847999215126038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,2,128,1,float16,float16,0,0.02146133283774058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,2,128,1,fp8,fp8,0,0.02515733242034912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,4,128,1,float16,float16,0,0.02143999934196472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,2,128,1,float16,fp8,0,0.021498667697111767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,4,128,1,float16,fp8,0,0.021503999829292297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,12,4,128,1,fp8,fp8,0,0.02508266766866048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,12,128,1,float16,float16,0,0.01773333301146825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,12,128,1,float16,fp8,0,0.018474667022625606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,1,128,1,float16,float16,0,0.017397332936525345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,12,128,1,fp8,fp8,0,0.020928000410397846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,1,128,1,fp8,fp8,0,0.020703999946514767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,1,128,1,float16,fp8,0,0.017935999979575474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,2,128,1,float16,float16,0,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,2,128,1,float16,fp8,0,0.017690667261679966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,2,128,1,fp8,fp8,0,0.020960000654061634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,4,128,1,float16,fp8,0,0.017909333109855652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,4,128,1,float16,float16,0,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,12,4,128,1,fp8,fp8,0,0.021002667645613354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,12,128,1,float16,float16,0,0.014869333555301031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,12,128,1,fp8,fp8,0,0.01714133347074191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,1,128,1,float16,float16,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,12,128,1,float16,fp8,0,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,1,128,1,fp8,fp8,0,0.01670933390657107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,1,128,1,float16,fp8,0,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,2,128,1,float16,float16,0,0.014389333625634512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,2,128,1,float16,fp8,0,0.014949332922697067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,2,128,1,fp8,fp8,0,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,4,128,1,float16,float16,0,0.014335999886194864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,12,128,1,float16,float16,0,0.014053333550691605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,4,128,1,float16,fp8,0,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,12,4,128,1,fp8,fp8,0,0.016885332763195038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,12,128,1,float16,fp8,0,0.01463466634353002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,12,128,1,fp8,fp8,0,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,1,128,1,float16,float16,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,2,128,1,float16,float16,0,0.01421333352724711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,1,128,1,float16,fp8,0,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,1,128,1,fp8,fp8,0,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,2,128,1,float16,fp8,0,0.014725333700577417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,2,128,1,fp8,fp8,0,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,4,128,1,float16,float16,0,0.014479999740918478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,4,128,1,float16,fp8,0,0.014698666830857595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,12,4,128,1,fp8,fp8,0,0.01664000004529953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,12,128,1,float16,float16,0,0.013834666460752487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,12,128,1,fp8,fp8,0,0.015856000284353893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,12,128,1,float16,fp8,0,0.014458666245142618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,1,128,1,float16,float16,0,0.014117332796255747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,1,128,1,float16,fp8,0,0.014720000326633453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,1,128,1,fp8,fp8,0,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,2,128,1,float16,float16,0,0.014085333794355392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,2,128,1,fp8,fp8,0,0.016250666230916977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,2,128,1,float16,fp8,0,0.014783999572197596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,4,128,1,float16,float16,0,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,12,128,1,float16,float16,0,0.013744000345468521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,4,128,1,fp8,fp8,0,0.0162773331006368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,12,4,128,1,float16,fp8,0,0.014736000448465347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,12,128,1,float16,fp8,0,0.01414399966597557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,12,128,1,fp8,fp8,0,0.01588800052801768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,1,128,1,float16,float16,0,0.013893333574136099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,1,128,1,float16,fp8,0,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,2,128,1,float16,float16,0,0.013978666315476099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,1,128,1,fp8,fp8,0,0.0161013330022494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,2,128,1,float16,fp8,0,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,2,128,1,fp8,fp8,0,0.01617066686352094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,4,128,1,float16,float16,0,0.014106666048367819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,4,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,12,4,128,1,fp8,fp8,0,0.016058667252461117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,1,128,1,fp8,fp8,0,5.31438414255778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,1,128,1,float16,fp8,0,8.11578114827474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,1,128,1,float16,float16,0,8.265098571777344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,2,128,1,float16,float16,0,8.152506510416666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,2,128,1,fp8,fp8,0,5.0927785237630205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,2,128,1,float16,fp8,0,8.231760025024414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,4,128,1,float16,fp8,0,8.214853286743164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,4,128,1,float16,float16,0,8.185669581095377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,1,128,1,float16,float16,0,4.011290550231934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,1,128,1,fp8,fp8,0,2.6079626083374023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,8,128,1,float16,float16,0,4.1028852462768555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,1,128,1,float16,fp8,0,3.8980159759521484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,2,128,1,float16,float16,0,3.9768266677856445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,8,128,1,fp8,fp8,0,2.552896022796631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,8,128,1,float16,fp8,0,3.960778554280599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,2,128,1,float16,fp8,0,3.8897228240966797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,8,4,128,1,fp8,fp8,0,5.322736104329427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,2,128,1,fp8,fp8,0,2.625999927520752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,8,128,1,float16,float16,0,1.981770674387614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,1,128,1,float16,float16,0,2.0717652638753257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,4,128,1,float16,float16,0,4.036970774332683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,4,128,1,fp8,fp8,0,2.6349493662516275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,8,4,128,1,float16,fp8,0,3.9166879653930664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,8,128,1,fp8,fp8,0,1.3465973536173503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,8,128,1,float16,fp8,0,1.9814453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,1,128,1,float16,fp8,0,2.0329012870788574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,1,128,1,fp8,fp8,0,1.3766026496887207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,2,128,1,float16,float16,0,2.0794933636983237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,2,128,1,float16,fp8,0,2.045525391896566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,2,128,1,fp8,fp8,0,1.3814454078674316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,8,128,1,float16,float16,0,1.0909759998321533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,4,128,1,float16,fp8,0,2.039642651875814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,1,128,1,float16,float16,0,1.1215413411458333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,4,128,1,fp8,fp8,0,1.3920052846272786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,8,128,1,float16,fp8,0,1.076149304707845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,1,128,1,float16,fp8,0,1.108191967010498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,8,4,128,1,float16,float16,0,2.056986649831136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,8,128,1,fp8,fp8,0,0.6833813190460205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,1,128,1,fp8,fp8,0,0.7033119996388754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,2,128,1,float16,float16,0,1.1221973101298015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,2,128,1,float16,fp8,0,1.1174826622009277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,4,128,1,float16,float16,0,1.1205493609110515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,4,128,1,float16,fp8,0,1.1130773226420085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,4,128,1,fp8,fp8,0,0.7047946453094482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,8,2,128,1,fp8,fp8,0,0.7015999952952067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,1,128,1,float16,fp8,0,4.663317362467448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,1,128,1,float16,float16,0,4.482816060384114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,1,128,1,fp8,fp8,0,3.0075200398763022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,2,128,1,float16,float16,0,4.611194610595703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,2,128,1,fp8,fp8,0,3.019733428955078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,2,128,1,float16,fp8,0,4.603919982910156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,4,128,1,float16,float16,0,4.605114618937175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,4,128,1,float16,fp8,0,4.487807909647624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,1,128,1,float16,float16,0,2.296565373738607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,1,128,1,float16,fp8,0,2.2770187060038247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,8,128,1,float16,float16,0,2.274618625640869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,1,128,1,fp8,fp8,0,1.568954626719157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,8,128,1,fp8,fp8,0,1.5940213203430176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,8,128,1,float16,fp8,0,2.2607572873433432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,8,4,128,1,fp8,fp8,0,3.054229418436686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,2,128,1,fp8,fp8,0,1.5735732714335124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,2,128,1,float16,fp8,0,2.285173257191976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,2,128,1,float16,float16,0,2.3075413703918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,4,128,1,float16,float16,0,2.321536064147949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,8,128,1,float16,fp8,0,1.1993920008341472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,1,128,1,float16,float16,0,1.2201920350392659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,8,128,1,float16,float16,0,1.2042346795399983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,4,128,1,float16,fp8,0,2.3433705965677896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,8,4,128,1,fp8,fp8,0,1.62772798538208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,1,128,1,float16,fp8,0,1.2135413487752278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,8,128,1,fp8,fp8,0,0.7843733628590902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,1,128,1,fp8,fp8,0,0.7909119923909506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,2,128,1,float16,float16,0,1.2300426959991455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,2,128,1,float16,fp8,0,1.2163413365681965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,2,128,1,fp8,fp8,0,0.7959626515706381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,4,128,1,float16,float16,0,1.2481013139088948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,4,128,1,float16,fp8,0,1.2401493390401204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,8,128,1,float16,float16,0,0.5813813209533691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,8,4,128,1,fp8,fp8,0,0.8080906867980957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,1,128,1,float16,float16,0,0.5903679927190145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,8,128,1,float16,fp8,0,0.5818719863891602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,8,128,1,fp8,fp8,0,0.41896533966064453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,1,128,1,fp8,fp8,0,0.4272480010986328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,2,128,1,float16,float16,0,0.5964106718699137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,1,128,1,float16,fp8,0,0.5856906572977701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,2,128,1,float16,fp8,0,0.5867199897766113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,2,128,1,fp8,fp8,0,0.4277919928232829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,4,128,1,float16,float16,0,0.5966506799062093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,4,128,1,float16,fp8,0,0.591818650563558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,8,4,128,1,fp8,fp8,0,0.43357332547505695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,1,128,1,float16,fp8,0,3.2714665730794272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,1,128,1,float16,float16,0,3.3347946802775064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,1,128,1,fp8,fp8,0,2.179157257080078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,2,128,1,float16,float16,0,3.2666454315185547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,2,128,1,float16,fp8,0,3.2572959264119468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,2,128,1,fp8,fp8,0,2.188810666402181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,4,128,1,float16,fp8,0,3.221407890319824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,4,128,1,float16,float16,0,3.2454026540120444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,1,128,1,float16,fp8,0,1.6377493540445964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,1,128,1,float16,float16,0,1.6552160580952961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,8,128,1,float16,float16,0,1.663717269897461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,1,128,1,fp8,fp8,0,1.1410240332285564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,8,128,1,fp8,fp8,0,1.136303981145223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,8,4,128,1,fp8,fp8,0,2.2927467028299966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,8,128,1,float16,fp8,0,1.6299360593159993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,2,128,1,float16,float16,0,1.6622826258341472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,2,128,1,float16,fp8,0,1.64247465133667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,2,128,1,fp8,fp8,0,1.1461386680603027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,8,128,1,float16,float16,0,0.9056426684061686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,4,128,1,float16,float16,0,1.6975092887878418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,4,128,1,float16,fp8,0,1.6637439727783203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,8,4,128,1,fp8,fp8,0,1.160154660542806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,1,128,1,float16,float16,0,0.8903679847717285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,8,128,1,fp8,fp8,0,0.5777653455734253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,1,128,1,float16,fp8,0,0.8831040064493815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,8,128,1,float16,fp8,0,0.8742720286051432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,1,128,1,fp8,fp8,0,0.5866613388061523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,2,128,1,float16,fp8,0,0.8900266488393148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,2,128,1,float16,float16,0,0.9037173589070638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,2,128,1,fp8,fp8,0,0.581930677096049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,4,128,1,float16,float16,0,0.8969439665476481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,4,128,1,float16,fp8,0,0.8928960164388021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,8,128,1,float16,float16,0,0.432528018951416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,8,4,128,1,fp8,fp8,0,0.5883573293685913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,8,128,1,float16,fp8,0,0.4330720106760661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,1,128,1,float16,fp8,0,0.43357332547505695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,1,128,1,float16,float16,0,0.439248005549113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,1,128,1,fp8,fp8,0,0.32016533613204956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,2,128,1,float16,float16,0,0.4398133357365926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,8,128,1,fp8,fp8,0,0.31678932905197144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,2,128,1,fp8,fp8,0,0.3207893371582031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,4,128,1,float16,float16,0,0.4495199918746948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,2,128,1,float16,fp8,0,0.4360746542612712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,4,128,1,fp8,fp8,0,0.3234666585922241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,8,4,128,1,float16,fp8,0,0.4379040002822876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,1,128,1,fp8,fp8,0,2.8885227839152017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,1,128,1,float16,float16,0,4.220261255900065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,1,128,1,float16,fp8,0,4.14301872253418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,2,128,1,float16,float16,0,4.330842653910319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,2,128,1,float16,fp8,0,4.400954564412435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,2,128,1,fp8,fp8,0,2.896640141805013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,4,128,1,float16,fp8,0,4.232234636942546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,4,128,1,float16,float16,0,4.328559875488281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,1,128,1,float16,float16,0,2.220640023549398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,1,128,1,float16,fp8,0,2.123077392578125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,8,128,1,float16,float16,0,2.131237347920736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,8,128,1,float16,fp8,0,2.115861256917318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,1,128,1,fp8,fp8,0,1.4826292991638184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,2,128,1,float16,float16,0,2.186255931854248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,8,128,1,fp8,fp8,0,1.489909331003825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,8,4,128,1,fp8,fp8,0,2.9588905970255532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,2,128,1,float16,fp8,0,2.2131946881612143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,2,128,1,fp8,fp8,0,1.4938240051269531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,8,128,1,float16,float16,0,1.1259466807047527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,4,128,1,float16,float16,0,2.1636266708374023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,4,128,1,float16,fp8,0,2.1475146611531577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,1,128,1,float16,float16,0,1.1342879931132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,8,4,128,1,fp8,fp8,0,1.511583964029948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,8,128,1,fp8,fp8,0,0.7903839747111002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,8,128,1,float16,fp8,0,1.0999680360158284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,1,128,1,float16,fp8,0,1.1064746379852295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,1,128,1,fp8,fp8,0,0.7803200085957845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,2,128,1,float16,float16,0,1.1193173726399739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,2,128,1,float16,fp8,0,1.109498659769694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,2,128,1,fp8,fp8,0,0.8057759602864584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,4,128,1,float16,fp8,0,1.121498664220174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,4,128,1,float16,float16,0,1.126207987467448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,8,4,128,1,fp8,fp8,0,0.8001279830932617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,1,128,1,float16,fp8,0,0.6127680142720541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,8,128,1,float16,float16,0,0.6056053241093954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,1,128,1,float16,float16,0,0.613594651222229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,8,128,1,float16,fp8,0,0.6068053245544434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,1,128,1,fp8,fp8,0,0.40066667397816974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,8,128,1,fp8,fp8,0,0.40283199151357013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,2,128,1,float16,float16,0,0.6146080096562704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,2,128,1,float16,fp8,0,0.6094719966252645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,2,128,1,fp8,fp8,0,0.4100746711095174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,4,128,1,float16,float16,0,0.618341326713562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,8,128,1,float16,fp8,0,0.2897653381029765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,8,128,1,float16,float16,0,0.29497067133585614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,4,128,1,fp8,fp8,0,0.40757866700490314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,8,128,1,fp8,fp8,0,0.2217493255933126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,8,4,128,1,float16,fp8,0,0.6161439816157023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,1,128,1,float16,fp8,0,0.2914133270581563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,1,128,1,fp8,fp8,0,0.22127999862035116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,2,128,1,float16,fp8,0,0.2934826612472534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,1,128,1,float16,float16,0,0.29499733448028564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,2,128,1,float16,float16,0,0.2958986759185791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,2,128,1,fp8,fp8,0,0.22291199366251627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,4,128,1,float16,float16,0,0.2971573273340861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,4,128,1,float16,fp8,0,0.29449599981307983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,8,4,128,1,fp8,fp8,0,0.23043199380238852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,1,128,1,float16,float16,0,2.5247626304626465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,1,128,1,float16,fp8,0,2.4895520210266113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,2,128,1,float16,float16,0,2.5272480646769204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,1,128,1,fp8,fp8,0,1.7667733828226726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,2,128,1,float16,fp8,0,2.535914738972982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,2,128,1,fp8,fp8,0,1.7835839589436848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,4,128,1,float16,fp8,0,2.684805234273275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,4,128,1,float16,float16,0,2.564469337463379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,1,128,1,float16,float16,0,1.310090700785319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,1,128,1,float16,fp8,0,1.2708853085835774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,8,128,1,float16,float16,0,1.3091200192769368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,8,128,1,float16,fp8,0,1.283893346786499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,1,128,1,fp8,fp8,0,0.9534133275349935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,8,4,128,1,fp8,fp8,0,1.8140907287597656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,8,128,1,fp8,fp8,0,0.9352746804555258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,2,128,1,float16,fp8,0,1.2975839773813884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,2,128,1,float16,float16,0,1.3023146788279216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,2,128,1,fp8,fp8,0,0.9260533650716146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,4,128,1,float16,float16,0,1.3165706793467205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,1,128,1,float16,float16,0,0.6864213148752848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,8,128,1,float16,float16,0,0.6862080097198486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,8,128,1,float16,fp8,0,0.6791679859161377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,8,128,1,fp8,fp8,0,0.49012800057729083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,4,128,1,float16,fp8,0,1.2928053538004558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,1,128,1,float16,fp8,0,0.6755359967549642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,8,4,128,1,fp8,fp8,0,0.9944053490956625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,1,128,1,fp8,fp8,0,0.46874133745829266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,2,128,1,float16,float16,0,0.6897119681040446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,2,128,1,fp8,fp8,0,0.4713759819666545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,2,128,1,float16,fp8,0,0.6805973052978516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,8,128,1,float16,float16,0,0.3600746790568034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,4,128,1,float16,fp8,0,0.6873653729756674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,4,128,1,float16,float16,0,0.7012320359547933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,8,4,128,1,fp8,fp8,0,0.48525333404541016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,8,128,1,float16,fp8,0,0.33814934889475506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,1,128,1,float16,float16,0,0.33583998680114746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,1,128,1,float16,fp8,0,0.32997334003448486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,1,128,1,fp8,fp8,0,0.2593066692352295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,2,128,1,float16,float16,0,0.3375413417816162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,8,128,1,fp8,fp8,0,0.2596106727917989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,2,128,1,float16,fp8,0,0.3325973351796468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,2,128,1,fp8,fp8,0,0.2552479902903239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,4,128,1,float16,float16,0,0.3431253433227539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,8,128,1,float16,float16,0,0.19905600945154825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,1,128,1,float16,float16,0,0.19502933820088705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,4,128,1,fp8,fp8,0,0.25912533203760785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,8,4,128,1,float16,fp8,0,0.33824535210927326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,8,128,1,float16,fp8,0,0.1921280026435852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,1,128,1,float16,fp8,0,0.19234667221705118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,8,128,1,fp8,fp8,0,0.15190933148066202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,1,128,1,fp8,fp8,0,0.14575466513633728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,2,128,1,float16,float16,0,0.1965013345082601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,2,128,1,float16,fp8,0,0.1944213310877482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,4,128,1,float16,float16,0,0.1989120046297709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,4,128,1,float16,fp8,0,0.19550933440526327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,2,128,1,fp8,fp8,0,0.14850667119026184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,8,4,128,1,fp8,fp8,0,0.15315733353296915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,1,128,1,float16,fp8,0,2.450453281402588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,1,128,1,float16,float16,0,2.505429267883301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,1,128,1,fp8,fp8,0,1.9682079950968425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,2,128,1,float16,float16,0,2.563360055287679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,2,128,1,fp8,fp8,0,1.8263146082560222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,2,128,1,float16,fp8,0,2.4603093465169272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,4,128,1,float16,fp8,0,2.4856053988138833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,4,128,1,float16,float16,0,2.578752040863037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,1,128,1,float16,float16,0,1.2612106800079346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,1,128,1,float16,fp8,0,1.2386133670806885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,8,128,1,float16,fp8,0,1.394752025604248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,1,128,1,fp8,fp8,0,0.9234560330708822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,8,128,1,float16,float16,0,1.3003573417663574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,8,128,1,fp8,fp8,0,0.9695573647816976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,8,4,128,1,fp8,fp8,0,1.8781760533650715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,2,128,1,float16,float16,0,1.2737600008646648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,2,128,1,float16,fp8,0,1.2483466466267903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,2,128,1,fp8,fp8,0,0.9352426528930664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,1,128,1,float16,float16,0,0.6545333464940389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,8,128,1,float16,float16,0,0.6811573505401611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,4,128,1,float16,float16,0,1.4042186737060547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,4,128,1,float16,fp8,0,1.2653653621673584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,1,128,1,float16,fp8,0,0.6430666844050089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,8,128,1,float16,fp8,0,0.6633919874827067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,8,4,128,1,fp8,fp8,0,0.9574613571166992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,8,128,1,fp8,fp8,0,0.5053386688232422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,1,128,1,fp8,fp8,0,0.48417067527770996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,2,128,1,float16,float16,0,0.6651573181152344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,2,128,1,float16,fp8,0,0.6725172996520996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,2,128,1,fp8,fp8,0,0.48978666464487713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,4,128,1,float16,fp8,0,0.6593706607818604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,4,128,1,float16,float16,0,0.6701013247172037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,8,128,1,float16,float16,0,0.3642880121866862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,8,4,128,1,fp8,fp8,0,0.5019146601359049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,1,128,1,float16,float16,0,0.36611199378967285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,1,128,1,float16,fp8,0,0.3524266481399536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,8,128,1,float16,fp8,0,0.35942399501800537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,1,128,1,fp8,fp8,0,0.2505919933319092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,8,128,1,fp8,fp8,0,0.267359991868337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,2,128,1,float16,float16,0,0.36208001772562665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,2,128,1,float16,fp8,0,0.3576853275299072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,2,128,1,fp8,fp8,0,0.2534506718317668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,4,128,1,float16,float16,0,0.3834826548894246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,4,128,1,float16,fp8,0,0.3609600067138672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,8,4,128,1,fp8,fp8,0,0.25910399357477826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,8,128,1,float16,float16,0,0.18150399128595987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,8,128,1,float16,fp8,0,0.17997866868972778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,8,128,1,fp8,fp8,0,0.14628266294797262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,1,128,1,float16,float16,0,0.17589867115020752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,1,128,1,fp8,fp8,0,0.14215999841690063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,2,128,1,float16,float16,0,0.17677332957585654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,1,128,1,float16,fp8,0,0.17293866475423178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,2,128,1,float16,fp8,0,0.17409066359202066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,2,128,1,fp8,fp8,0,0.14125866691271463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,4,128,1,float16,float16,0,0.18079467614491782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,4,128,1,float16,fp8,0,0.17733865976333618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,8,4,128,1,fp8,fp8,0,0.14380799730618796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,1,128,1,float16,float16,0,0.09965333342552185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,8,128,1,float16,float16,0,0.10269332925478618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,1,128,1,float16,fp8,0,0.09888000289599101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,8,128,1,float16,fp8,0,0.0997866690158844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,8,128,1,fp8,fp8,0,0.08586666981379192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,1,128,1,fp8,fp8,0,0.07960533102353413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,2,128,1,float16,float16,0,0.10028266906738281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,2,128,1,float16,fp8,0,0.09897599617640178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,2,128,1,fp8,fp8,0,0.08006933331489563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,4,128,1,float16,fp8,0,0.10139200091362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,4,128,1,fp8,fp8,0,0.08297599852085114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,8,4,128,1,float16,float16,0,0.10146666566530864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,1,128,1,float16,fp8,0,1.5330665906270344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,1,128,1,float16,float16,0,1.5633440017700195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,1,128,1,fp8,fp8,0,1.1658506393432617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,2,128,1,fp8,fp8,0,1.1818293730417888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,2,128,1,float16,fp8,0,1.5300960540771484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,2,128,1,float16,float16,0,1.579194704691569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,4,128,1,float16,float16,0,1.6093333562215169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,4,128,1,float16,fp8,0,1.724239985148112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,1,128,1,float16,float16,0,0.7908533414204916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,1,128,1,float16,fp8,0,0.7729173501332601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,1,128,1,fp8,fp8,0,0.5971306562423706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,8,128,1,float16,fp8,0,0.805781364440918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,8,128,1,float16,float16,0,0.8279039859771729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,8,128,1,fp8,fp8,0,0.6368266741434733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,2,128,1,float16,float16,0,0.8561440308888754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,8,4,128,1,fp8,fp8,0,1.2086079915364583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,2,128,1,float16,fp8,0,0.7799893220265707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,2,128,1,fp8,fp8,0,0.6069493293762207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,1,128,1,float16,float16,0,0.41872533162434894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,4,128,1,float16,float16,0,0.8092160224914551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,8,128,1,float16,float16,0,0.4346933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,8,128,1,float16,fp8,0,0.4697173436482747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,1,128,1,float16,fp8,0,0.4089226722717285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,4,128,1,float16,fp8,0,0.794490655263265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,8,128,1,fp8,fp8,0,0.3397119839986165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,8,4,128,1,fp8,fp8,0,0.6243573427200317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,1,128,1,fp8,fp8,0,0.30447999636332196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,2,128,1,float16,fp8,0,0.4139680067698161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,2,128,1,fp8,fp8,0,0.3248639901479085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,2,128,1,float16,float16,0,0.420634667078654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,4,128,1,float16,float16,0,0.4270293315251668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,4,128,1,fp8,fp8,0,0.3235573371251424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,8,4,128,1,float16,fp8,0,0.4200640122095744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,1,128,1,float16,fp8,0,0.20505066712697348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,1,128,1,float16,float16,0,0.21500800053278604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,8,128,1,float16,fp8,0,0.21744000911712646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,8,128,1,float16,float16,0,0.22241065899531046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,8,128,1,fp8,fp8,0,0.1795733372370402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,1,128,1,fp8,fp8,0,0.1657546659310659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,2,128,1,float16,float16,0,0.21197867393493652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,2,128,1,float16,fp8,0,0.20768000682195029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,2,128,1,fp8,fp8,0,0.16760534048080444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,8,128,1,float16,float16,0,0.12294399738311768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,4,128,1,float16,float16,0,0.23163199424743652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,4,128,1,float16,fp8,0,0.2134666641553243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,8,128,1,float16,fp8,0,0.12076266606648763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,8,128,1,fp8,fp8,0,0.10207999746004741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,8,4,128,1,fp8,fp8,0,0.1739679972330729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,1,128,1,float16,float16,0,0.11754666765530904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,1,128,1,float16,fp8,0,0.11565867066383362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,1,128,1,fp8,fp8,0,0.09216533104578654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,2,128,1,float16,float16,0,0.11832533280054729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,2,128,1,float16,fp8,0,0.11773866415023804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,2,128,1,fp8,fp8,0,0.09539199868837993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,4,128,1,float16,float16,0,0.12138666709264119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,8,128,1,float16,float16,0,0.07457066575686137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,4,128,1,fp8,fp8,0,0.09990400075912476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,8,4,128,1,float16,fp8,0,0.11991999546686809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,8,128,1,float16,fp8,0,0.07423999905586243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,1,128,1,float16,float16,0,0.07444266478220622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,8,128,1,fp8,fp8,0,0.06443733473618825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,1,128,1,fp8,fp8,0,0.06049066781997681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,1,128,1,float16,fp8,0,0.07369066774845123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,2,128,1,float16,float16,0,0.07451733450094859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,4,128,1,float16,float16,0,0.0749066670735677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,2,128,1,float16,fp8,0,0.07375999788443248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,2,128,1,fp8,fp8,0,0.06112533311049143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,4,128,1,fp8,fp8,0,0.061610668897628784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,8,4,128,1,float16,fp8,0,0.07412266731262207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,1,128,1,float16,float16,0,1.6868906021118164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,1,128,1,float16,fp8,0,1.6068159739176433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,1,128,1,fp8,fp8,0,1.3076746463775635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,2,128,1,float16,float16,0,1.6980640093485515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,2,128,1,float16,fp8,0,1.6284747123718262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,2,128,1,fp8,fp8,0,1.3281066417694092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,4,128,1,float16,fp8,0,1.6595199902852376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,4,128,1,float16,float16,0,1.9820106824239094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,1,128,1,float16,float16,0,0.8459680080413818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,1,128,1,float16,fp8,0,0.8124213218688965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,1,128,1,fp8,fp8,0,0.6632533470789591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,8,128,1,fp8,fp8,0,0.716538667678833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,2,128,1,float16,float16,0,0.9458186626434326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,8,128,1,float16,float16,0,0.903061310450236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,8,128,1,float16,fp8,0,0.8615413506825765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,8,4,128,1,fp8,fp8,0,1.3580479621887207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,2,128,1,float16,fp8,0,0.8213013013203939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,2,128,1,fp8,fp8,0,0.6688213348388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,1,128,1,float16,float16,0,0.4286133448282878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,4,128,1,float16,float16,0,0.8746986389160156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,8,128,1,float16,fp8,0,0.4455893437067668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,8,128,1,float16,float16,0,0.4585173527399699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,4,128,1,float16,fp8,0,0.8421066602071127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,8,128,1,fp8,fp8,0,0.37328533331553143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,8,4,128,1,fp8,fp8,0,0.767520030339559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,1,128,1,float16,fp8,0,0.4176266590754191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,1,128,1,fp8,fp8,0,0.34118398030598956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,2,128,1,fp8,fp8,0,0.34594666957855225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,2,128,1,float16,fp8,0,0.4211946725845337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,2,128,1,float16,float16,0,0.43481600284576416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,4,128,1,float16,float16,0,0.4449973503748576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,4,128,1,float16,fp8,0,0.4341760079065959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,8,4,128,1,fp8,fp8,0,0.3909226655960083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,8,128,1,float16,float16,0,0.24699733654658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,1,128,1,float16,float16,0,0.2310346762339274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,8,128,1,float16,fp8,0,0.24100265900293985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,8,128,1,fp8,fp8,0,0.19698133071263632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,1,128,1,float16,fp8,0,0.2264159917831421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,1,128,1,fp8,fp8,0,0.1777013341585795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,2,128,1,float16,float16,0,0.23498666286468506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,2,128,1,float16,fp8,0,0.23966399828592935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,2,128,1,fp8,fp8,0,0.18057066202163696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,4,128,1,float16,float16,0,0.23859200874964395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,8,128,1,float16,fp8,0,0.12557866175969443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,8,128,1,float16,float16,0,0.12638399998346964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,4,128,1,fp8,fp8,0,0.18530132373174033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,8,128,1,fp8,fp8,0,0.10814399520556132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,8,4,128,1,float16,fp8,0,0.23449599742889404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,1,128,1,float16,float16,0,0.11967999736467998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,2,128,1,float16,float16,0,0.11932800213495891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,1,128,1,fp8,fp8,0,0.09783466657002766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,1,128,1,float16,fp8,0,0.11538666486740112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,4,128,1,float16,float16,0,0.12179199854532878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,2,128,1,float16,fp8,0,0.11610666910807292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,2,128,1,fp8,fp8,0,0.09963200489679973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,4,128,1,float16,fp8,0,0.11989333232243855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,8,4,128,1,fp8,fp8,0,0.10951466361681621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,8,128,1,float16,float16,0,0.07007466753323872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,8,128,1,float16,fp8,0,0.0695306658744812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,1,128,1,float16,float16,0,0.06600533425807953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,2,128,1,float16,float16,0,0.0662613312403361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,2,128,1,float16,fp8,0,0.06534933547178905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,8,128,1,fp8,fp8,0,0.06398400167624156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,1,128,1,float16,fp8,0,0.06485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,1,128,1,fp8,fp8,0,0.055120001236597695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,2,128,1,fp8,fp8,0,0.055973331133524575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,4,128,1,float16,float16,0,0.06769066552321117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,4,128,1,float16,fp8,0,0.06724800169467926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,8,4,128,1,fp8,fp8,0,0.05861333509286245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,8,128,1,float16,float16,0,0.05342400074005127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,1,128,1,float16,float16,0,0.05442133545875549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,1,128,1,fp8,fp8,0,0.04456533491611481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,1,128,1,float16,fp8,0,0.05358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,2,128,1,float16,float16,0,0.05420266588528951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,8,128,1,float16,fp8,0,0.053488001227378845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,8,128,1,fp8,fp8,0,0.045461331804593406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,2,128,1,fp8,fp8,0,0.04494399825731913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,2,128,1,float16,fp8,0,0.05386666456858317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,4,128,1,float16,float16,0,0.0545653353134791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,4,128,1,float16,fp8,0,0.0539626677831014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,8,4,128,1,fp8,fp8,0,0.04569066564242045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,1,128,1,float16,float16,0,1.085034688313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,1,128,1,float16,fp8,0,1.0231733322143555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,2,128,1,fp8,fp8,0,0.887498696645101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,2,128,1,float16,float16,0,1.2398666540781658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,2,128,1,float16,fp8,0,1.033344030380249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,1,128,1,fp8,fp8,0,0.8852693239847819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,4,128,1,float16,fp8,0,1.061797300974528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,4,128,1,float16,float16,0,1.1348106861114502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,1,128,1,fp8,fp8,0,0.4484746853510539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,1,128,1,float16,fp8,0,0.5249706506729126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,1,128,1,float16,float16,0,0.5511360168457031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,2,128,1,float16,float16,0,0.5573920011520386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,8,128,1,fp8,fp8,0,0.5668479998906454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,8,4,128,1,fp8,fp8,0,0.9170400301615397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,8,128,1,float16,fp8,0,0.5698186556498209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,8,128,1,float16,float16,0,0.5943093299865723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,2,128,1,float16,fp8,0,0.5334240198135376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,2,128,1,fp8,fp8,0,0.4580693244934082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,8,128,1,float16,float16,0,0.3108853300412496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,1,128,1,float16,float16,0,0.287663996219635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,4,128,1,fp8,fp8,0,0.5217920144399008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,4,128,1,float16,float16,0,0.5748159885406494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,8,4,128,1,float16,fp8,0,0.5419360001881918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,8,128,1,float16,fp8,0,0.3007413347562154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,1,128,1,float16,fp8,0,0.27745066086451214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,8,128,1,fp8,fp8,0,0.27613333861033124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,1,128,1,fp8,fp8,0,0.22664533058802286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,2,128,1,float16,float16,0,0.2913866639137268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,2,128,1,float16,fp8,0,0.2986026604970296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,2,128,1,fp8,fp8,0,0.23196266094843546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,4,128,1,float16,fp8,0,0.2860213319460551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,4,128,1,float16,float16,0,0.29678932825724286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,1,128,1,float16,float16,0,0.15407466888427734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,8,4,128,1,fp8,fp8,0,0.24336000283559164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,8,128,1,fp8,fp8,0,0.13896000385284424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,8,128,1,float16,fp8,0,0.16580800215403238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,8,128,1,float16,float16,0,0.17017066478729248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,1,128,1,fp8,fp8,0,0.12406933307647705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,1,128,1,float16,fp8,0,0.1451359987258911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,2,128,1,float16,float16,0,0.1497813363869985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,2,128,1,float16,fp8,0,0.15242133537928262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,2,128,1,fp8,fp8,0,0.12570133805274963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,4,128,1,float16,float16,0,0.15658666690190634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,4,128,1,float16,fp8,0,0.15215466419855753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,8,4,128,1,fp8,fp8,0,0.13210666179656982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,8,128,1,float16,float16,0,0.08804800113042195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,1,128,1,float16,float16,0,0.08010133107503255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,8,128,1,fp8,fp8,0,0.07922666768232982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,8,128,1,float16,fp8,0,0.08708266417185466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,1,128,1,float16,fp8,0,0.07834133505821228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,1,128,1,fp8,fp8,0,0.06769066552321117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,2,128,1,float16,float16,0,0.08190399905045827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,4,128,1,float16,float16,0,0.08737599849700928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,4,128,1,float16,fp8,0,0.08286933104197185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,8,128,1,float16,float16,0,0.05121066669623057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,2,128,1,fp8,fp8,0,0.07121066749095917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,8,128,1,float16,fp8,0,0.05027199784914652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,2,128,1,float16,fp8,0,0.07976533472537994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,8,128,1,fp8,fp8,0,0.04706666866938273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,8,4,128,1,fp8,fp8,0,0.07604800164699554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,1,128,1,float16,float16,0,0.049413333336512245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,1,128,1,float16,fp8,0,0.04778666794300079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,1,128,1,fp8,fp8,0,0.04192000130812327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,2,128,1,float16,fp8,0,0.04894933104515076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,2,128,1,float16,float16,0,0.049322664737701416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,4,128,1,float16,float16,0,0.04993066688378652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,2,128,1,fp8,fp8,0,0.04233066737651825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,8,128,1,float16,float16,0,0.043568000197410583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,4,128,1,float16,fp8,0,0.04903466502825419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,8,4,128,1,fp8,fp8,0,0.04393066465854645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,8,128,1,float16,fp8,0,0.04339733223120371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,1,128,1,float16,float16,0,0.04427733520666758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,8,128,1,fp8,fp8,0,0.03750933210055033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,1,128,1,float16,fp8,0,0.04354666670163473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,2,128,1,float16,float16,0,0.044138665000597634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,2,128,1,fp8,fp8,0,0.0373279998699824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,4,128,1,float16,fp8,0,0.04390933116277059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,4,128,1,float16,float16,0,0.04424533247947693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,4,128,1,fp8,fp8,0,0.03756800045569738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,2,128,1,float16,fp8,0,0.04394666850566864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,8,1,128,1,fp8,fp8,0,0.03716800113519033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,1,128,1,float16,float16,0,0.9935999711354574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,1,128,1,float16,fp8,0,0.9875679810841879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,1,128,1,fp8,fp8,0,0.862714687983195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,2,128,1,float16,fp8,0,1.0061546961466472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,2,128,1,fp8,fp8,0,0.8847040335337321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,2,128,1,float16,float16,0,1.014965295791626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,4,128,1,float16,fp8,0,1.0598986943562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,4,128,1,float16,float16,0,1.079914649327596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,1,128,1,float16,float16,0,0.509984016418457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,1,128,1,float16,fp8,0,0.5068639914194742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,8,128,1,float16,fp8,0,0.7720479965209961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,8,128,1,float16,float16,0,0.6270399888356527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,8,4,128,1,fp8,fp8,0,0.9810453255971273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,8,128,1,fp8,fp8,0,0.531930685043335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,1,128,1,fp8,fp8,0,0.42156267166137695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,2,128,1,float16,float16,0,0.5198293526967367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,2,128,1,float16,fp8,0,0.5189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,2,128,1,fp8,fp8,0,0.4434239864349365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,1,128,1,float16,float16,0,0.26826133330663043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,4,128,1,float16,float16,0,0.6707839965820312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,4,128,1,float16,fp8,0,0.542085329691569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,8,128,1,float16,float16,0,0.3234399954477946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,1,128,1,float16,fp8,0,0.26630399624506634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,8,4,128,1,fp8,fp8,0,0.4999093214670817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,8,128,1,float16,fp8,0,0.31355732679367065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,8,128,1,fp8,fp8,0,0.272271990776062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,1,128,1,fp8,fp8,0,0.22195732593536377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,2,128,1,float16,float16,0,0.3104959925015767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,2,128,1,float16,fp8,0,0.27112533648808795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,2,128,1,fp8,fp8,0,0.2300586700439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,4,128,1,float16,fp8,0,0.28487465778986615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,4,128,1,float16,float16,0,0.2909440000851949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,8,128,1,float16,fp8,0,0.16979734102884927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,1,128,1,float16,float16,0,0.1453973352909088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,8,128,1,float16,float16,0,0.17384000619252524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,1,128,1,float16,fp8,0,0.14591466387112936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,8,128,1,fp8,fp8,0,0.18021865685780844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,8,4,128,1,fp8,fp8,0,0.2572000026702881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,1,128,1,fp8,fp8,0,0.11385599772135417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,2,128,1,float16,float16,0,0.14825066924095154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,2,128,1,float16,fp8,0,0.14839466412862143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,2,128,1,fp8,fp8,0,0.1174773375193278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,4,128,1,float16,fp8,0,0.15533333023389181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,4,128,1,float16,float16,0,0.1585919956366221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,8,128,1,float16,fp8,0,0.09165333708127339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,8,4,128,1,fp8,fp8,0,0.14431466658910116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,1,128,1,float16,float16,0,0.0777759999036789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,8,128,1,float16,float16,0,0.09494400024414062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,1,128,1,float16,fp8,0,0.07768533130486806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,8,128,1,fp8,fp8,0,0.07801066835721333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,1,128,1,fp8,fp8,0,0.06406400104363759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,2,128,1,float16,fp8,0,0.07876800000667572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,2,128,1,float16,float16,0,0.07939733564853668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,2,128,1,fp8,fp8,0,0.07089066505432129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,4,128,1,float16,float16,0,0.08418132861455281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,4,128,1,fp8,fp8,0,0.0703359991312027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,8,4,128,1,float16,fp8,0,0.08228266735871632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,8,128,1,float16,float16,0,0.04916800061861674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,8,128,1,float16,fp8,0,0.04855466882387797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,8,128,1,fp8,fp8,0,0.045498669147491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,1,128,1,float16,fp8,0,0.04338666796684265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,1,128,1,fp8,fp8,0,0.036202666660149894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,1,128,1,float16,float16,0,0.043194666504859924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,2,128,1,float16,float16,0,0.04378666480382284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,4,128,1,float16,float16,0,0.04684799909591675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,2,128,1,float16,fp8,0,0.04387199878692627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,2,128,1,fp8,fp8,0,0.03757333258787791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,4,128,1,float16,fp8,0,0.04555733501911163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,8,4,128,1,fp8,fp8,0,0.040709334115187325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,8,128,1,float16,float16,0,0.03347733368476232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,8,128,1,fp8,fp8,0,0.02926933268706004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,8,128,1,float16,fp8,0,0.033674667278925575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,1,128,1,float16,float16,0,0.03221333275238673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,2,128,1,float16,float16,0,0.0322826678554217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,2,128,1,float16,fp8,0,0.03260799994071325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,2,128,1,fp8,fp8,0,0.02792533238728841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,1,128,1,float16,fp8,0,0.032101333141326904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,1,128,1,fp8,fp8,0,0.027471999327341717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,4,128,1,float16,float16,0,0.03326933334271113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,4,128,1,fp8,fp8,0,0.0288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,8,128,1,float16,float16,0,0.029616000751654308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,8,128,1,float16,fp8,0,0.029504001140594482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,8,128,1,fp8,fp8,0,0.025173333783944447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,8,4,128,1,float16,fp8,0,0.033285332222779594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,1,128,1,float16,fp8,0,0.029338667790095013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,1,128,1,float16,float16,0,0.029205332199732464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,1,128,1,fp8,fp8,0,0.024634666740894318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,2,128,1,float16,fp8,0,0.029535998900731403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,2,128,1,float16,float16,0,0.02943466603755951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,2,128,1,fp8,fp8,0,0.024890666206677754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,4,128,1,float16,float16,0,0.02985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,4,128,1,float16,fp8,0,0.029722665747006733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,8,4,128,1,fp8,fp8,0,0.025216000775496166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,1,128,1,float16,fp8,0,0.7764159838358561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,1,128,1,fp8,fp8,0,0.6886719862620035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,1,128,1,float16,float16,0,0.7827626864115397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,2,128,1,float16,float16,0,0.8045173486073812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,2,128,1,float16,fp8,0,0.9519519805908203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,2,128,1,fp8,fp8,0,0.7426293690999349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,4,128,1,float16,float16,0,0.8685119946797689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,1,128,1,float16,float16,0,0.4590080181757609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,1,128,1,float16,fp8,0,0.40097065766652423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,4,128,1,float16,fp8,0,0.8499840100606283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,8,128,1,float16,float16,0,0.5302879810333252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,1,128,1,fp8,fp8,0,0.3523999849955241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,8,128,1,fp8,fp8,0,0.4599573214848836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,2,128,1,float16,float16,0,0.49668800830841064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,2,128,1,float16,fp8,0,0.4102826515833537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,8,4,128,1,fp8,fp8,0,0.8400853474934896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,8,128,1,float16,fp8,0,0.5048693418502808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,2,128,1,fp8,fp8,0,0.3710879882176717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,8,128,1,float16,float16,0,0.39025600751241046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,8,128,1,float16,fp8,0,0.2639999985694885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,4,128,1,fp8,fp8,0,0.4329386552174886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,4,128,1,float16,float16,0,0.44660266240437824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,1,128,1,float16,float16,0,0.21042666832605997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,8,128,1,fp8,fp8,0,0.23666133483250937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,8,4,128,1,float16,fp8,0,0.4365653196970622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,1,128,1,float16,fp8,0,0.20977065960566202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,1,128,1,fp8,fp8,0,0.19948800404866537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,2,128,1,float16,float16,0,0.21724800268809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,2,128,1,float16,fp8,0,0.21463465690612793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,2,128,1,fp8,fp8,0,0.19212265809377035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,4,128,1,float16,fp8,0,0.22933866580327353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,4,128,1,float16,float16,0,0.23338133096694946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,8,128,1,float16,float16,0,0.14783466855684915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,8,4,128,1,fp8,fp8,0,0.21932266155878702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,1,128,1,float16,float16,0,0.11573867003122966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,1,128,1,float16,fp8,0,0.11514666676521301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,8,128,1,float16,fp8,0,0.19681066274642944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,8,128,1,fp8,fp8,0,0.12702932953834534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,1,128,1,fp8,fp8,0,0.09643733501434326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,4,128,1,float16,float16,0,0.12735467155774435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,2,128,1,float16,fp8,0,0.11786133050918579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,4,128,1,float16,fp8,0,0.12582932909329733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,2,128,1,float16,float16,0,0.11884267131487529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,4,128,1,fp8,fp8,0,0.12506133317947388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,8,2,128,1,fp8,fp8,0,0.09950400392214458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,8,128,1,float16,float16,0,0.08460799853006999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,1,128,1,float16,fp8,0,0.06467733283837636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,8,128,1,fp8,fp8,0,0.06898666421572368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,2,128,1,float16,float16,0,0.070933332045873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,8,128,1,float16,fp8,0,0.08102400104204814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,1,128,1,fp8,fp8,0,0.05487466851870219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,2,128,1,fp8,fp8,0,0.05840000013510386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,2,128,1,float16,fp8,0,0.06756799916426341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,1,128,1,float16,float16,0,0.06410666803518932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,4,128,1,float16,float16,0,0.07039466500282288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,4,128,1,fp8,fp8,0,0.07170133292675018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,8,4,128,1,float16,fp8,0,0.06901866694291432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,8,128,1,float16,float16,0,0.043493335445721946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,8,128,1,fp8,fp8,0,0.04136000076929728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,1,128,1,float16,float16,0,0.03721600025892258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,8,128,1,float16,fp8,0,0.0422026664018631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,1,128,1,float16,fp8,0,0.03730133424202601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,2,128,1,float16,float16,0,0.03783999880154928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,2,128,1,float16,fp8,0,0.03786666691303253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,2,128,1,fp8,fp8,0,0.03316266586383184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,1,128,1,fp8,fp8,0,0.03238933285077413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,4,128,1,float16,float16,0,0.04025600105524063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,4,128,1,float16,fp8,0,0.039247999588648476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,1,128,1,float16,float16,0,0.025978667040665943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,8,128,1,float16,float16,0,0.028394666810830433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,8,128,1,fp8,fp8,0,0.025018667181332905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,8,128,1,float16,fp8,0,0.02699200063943863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,1,128,1,fp8,fp8,0,0.023120000958442688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,1,128,1,float16,fp8,0,0.025424001117547352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,8,4,128,1,fp8,fp8,0,0.0367253323396047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,2,128,1,float16,fp8,0,0.026309333741664886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,4,128,1,float16,float16,0,0.027402666707833607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,4,128,1,float16,fp8,0,0.027466667195161183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,2,128,1,float16,float16,0,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,4,128,1,fp8,fp8,0,0.02489600082238515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,8,128,1,float16,float16,0,0.023647998770078022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,8,2,128,1,fp8,fp8,0,0.023567999402681988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,8,128,1,float16,fp8,0,0.023786666492621105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,1,128,1,float16,float16,0,0.02292799949645996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,8,128,1,fp8,fp8,0,0.021317332983016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,1,128,1,float16,fp8,0,0.02317333221435547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,2,128,1,float16,float16,0,0.02314666658639908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,2,128,1,float16,fp8,0,0.023168000082174938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,1,128,1,fp8,fp8,0,0.020389333367347717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,4,128,1,float16,float16,0,0.023573334018389385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,4,128,1,float16,fp8,0,0.023589332898457844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,2,128,1,fp8,fp8,0,0.020794666061798733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,8,128,1,float16,float16,0,0.021722666919231415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,8,128,1,float16,fp8,0,0.021802666286627453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,8,4,128,1,fp8,fp8,0,0.021365332106749218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,8,128,1,fp8,fp8,0,0.019274666905403137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,1,128,1,fp8,fp8,0,0.01915733392039935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,1,128,1,float16,fp8,0,0.021685334543387096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,2,128,1,float16,fp8,0,0.021935999393463135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,1,128,1,float16,float16,0,0.021712000171343487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,2,128,1,fp8,fp8,0,0.019343999524911244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,2,128,1,float16,float16,0,0.021568000316619873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,4,128,1,float16,float16,0,0.021754667162895203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,4,128,1,float16,fp8,0,0.0220320001244545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,8,4,128,1,fp8,fp8,0,0.019733333339293797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,1,128,1,float16,fp8,0,0.41468799114227295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,1,128,1,float16,float16,0,0.3456639846165975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,2,128,1,float16,float16,0,0.35784534613291424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,1,128,1,fp8,fp8,0,0.2988800009091695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,2,128,1,fp8,fp8,0,0.3275306622187297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,2,128,1,float16,fp8,0,0.35470398267110187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,4,128,1,float16,fp8,0,0.38091198603312176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,4,128,1,float16,float16,0,0.3935306469599406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,1,128,1,float16,float16,0,0.1818293333053589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,8,128,1,float16,float16,0,0.2553013364473979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,1,128,1,fp8,fp8,0,0.157669335603714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,8,128,1,float16,fp8,0,0.2434613307317098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,8,4,128,1,fp8,fp8,0,0.4474240144093831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,1,128,1,float16,fp8,0,0.18184532721837363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,8,128,1,fp8,fp8,0,0.21341866254806519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,2,128,1,float16,float16,0,0.1880213419596354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,2,128,1,float16,fp8,0,0.18693333864212036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,2,128,1,fp8,fp8,0,0.16690133015314737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,4,128,1,float16,float16,0,0.20569600661595663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,4,128,1,float16,fp8,0,0.20279999574025473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,8,4,128,1,fp8,fp8,0,0.23328532775243124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,8,128,1,fp8,fp8,0,0.1172213355700175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,1,128,1,float16,float16,0,0.10176533460617065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,8,128,1,float16,float16,0,0.13952533404032388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,8,128,1,float16,fp8,0,0.13429866234461466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,1,128,1,float16,fp8,0,0.1016319990158081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,1,128,1,fp8,fp8,0,0.08716266353925069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,2,128,1,float16,float16,0,0.10641599694887798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,2,128,1,float16,fp8,0,0.1285599966843923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,2,128,1,fp8,fp8,0,0.09029866258303325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,4,128,1,float16,float16,0,0.11512000362078349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,4,128,1,fp8,fp8,0,0.10487467050552368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,8,4,128,1,float16,fp8,0,0.1130506694316864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,1,128,1,float16,float16,0,0.058965335289637245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,1,128,1,float16,fp8,0,0.05783999959627787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,8,128,1,float16,fp8,0,0.0791786660750707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,8,128,1,fp8,fp8,0,0.06560533245404561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,8,128,1,float16,float16,0,0.08200533191363017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,1,128,1,fp8,fp8,0,0.05016533533732096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,2,128,1,fp8,fp8,0,0.05794133245944977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,4,128,1,float16,float16,0,0.06460266808668773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,4,128,1,fp8,fp8,0,0.05689600110054016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,4,128,1,float16,fp8,0,0.06320000191529591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,2,128,1,float16,float16,0,0.06046399970849355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,8,2,128,1,float16,fp8,0,0.05978133281071981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,8,128,1,float16,fp8,0,0.039408000806967415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,8,128,1,float16,float16,0,0.04064533362785975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,1,128,1,float16,float16,0,0.03468266626199087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,1,128,1,float16,fp8,0,0.034832000732421875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,8,128,1,fp8,fp8,0,0.04432533184687296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,1,128,1,fp8,fp8,0,0.0305173322558403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,2,128,1,fp8,fp8,0,0.030928000807762146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,2,128,1,float16,float16,0,0.03548266738653183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,4,128,1,float16,float16,0,0.03722666700681051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,4,128,1,float16,fp8,0,0.03675200045108795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,2,128,1,float16,fp8,0,0.035402665535608925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,8,4,128,1,fp8,fp8,0,0.03463999927043915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,8,128,1,float16,float16,0,0.025642665723959606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,8,128,1,float16,fp8,0,0.025237334271272022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,1,128,1,float16,float16,0,0.02312533309062322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,8,128,1,fp8,fp8,0,0.023317334552605946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,1,128,1,fp8,fp8,0,0.021216000119845074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,2,128,1,float16,fp8,0,0.023605334262053173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,1,128,1,float16,fp8,0,0.023530667026837666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,2,128,1,float16,float16,0,0.02418133368094762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,2,128,1,fp8,fp8,0,0.021685334543387096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,4,128,1,float16,float16,0,0.02489600082238515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,4,128,1,fp8,fp8,0,0.022698665658632915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,8,4,128,1,float16,fp8,0,0.024341332415739696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,8,128,1,float16,float16,0,0.02025066688656807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,8,128,1,float16,fp8,0,0.02022933339079221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,1,128,1,float16,fp8,0,0.019589333484570186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,1,128,1,float16,float16,0,0.01966933285196622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,8,128,1,fp8,fp8,0,0.019248000035683315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,2,128,1,float16,float16,0,0.019621333728233974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,2,128,1,float16,fp8,0,0.01979200045267741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,2,128,1,fp8,fp8,0,0.01858666663368543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,1,128,1,fp8,fp8,0,0.01834133391578992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,4,128,1,float16,float16,0,0.020015999674797058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,4,128,1,float16,fp8,0,0.020186666399240494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,8,4,128,1,fp8,fp8,0,0.019178666174411774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,8,128,1,float16,fp8,0,0.01870399961868922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,8,128,1,fp8,fp8,0,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,8,128,1,float16,float16,0,0.01863466699918111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,1,128,1,float16,float16,0,0.018239999810854595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,1,128,1,fp8,fp8,0,0.01708799973130226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,2,128,1,float16,float16,0,0.018309333672126133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,1,128,1,float16,fp8,0,0.018496000518401463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,2,128,1,fp8,fp8,0,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,2,128,1,float16,fp8,0,0.01852799952030182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,4,128,1,float16,float16,0,0.018613333503405254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,4,128,1,fp8,fp8,0,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,8,4,128,1,float16,fp8,0,0.0189280000825723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,8,128,1,float16,float16,0,0.017973333597183228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,8,128,1,float16,fp8,0,0.01846933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,8,128,1,fp8,fp8,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,1,128,1,float16,float16,0,0.017680000513792038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,1,128,1,fp8,fp8,0,0.017008000363906223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,1,128,1,float16,fp8,0,0.01834133391578992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,2,128,1,float16,float16,0,0.017680000513792038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,2,128,1,fp8,fp8,0,0.016997333616018295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,4,128,1,float16,float16,0,0.017770666629076004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,4,128,1,float16,fp8,0,0.018122666825850803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,4,128,1,fp8,fp8,0,0.016895999511082966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,8,2,128,1,float16,fp8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,1,128,1,fp8,fp8,0,0.15777599811553955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,1,128,1,float16,fp8,0,0.17495467265446982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,1,128,1,float16,float16,0,0.17493865887324014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,2,128,1,float16,float16,0,0.1811093290646871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,2,128,1,float16,fp8,0,0.2351199984550476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,2,128,1,fp8,fp8,0,0.16759467124938965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,4,128,1,float16,float16,0,0.20075732469558716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,4,128,1,float16,fp8,0,0.19484267632166544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,1,128,1,float16,float16,0,0.11893332997957866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,1,128,1,float16,fp8,0,0.09909866253534953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,8,4,128,1,fp8,fp8,0,0.19589332739512125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,8,128,1,fp8,fp8,0,0.11105066537857056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,1,128,1,fp8,fp8,0,0.0867146650950114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,8,128,1,float16,fp8,0,0.13779733578364053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,2,128,1,float16,float16,0,0.10338133573532104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,8,128,1,float16,float16,0,0.14412800470987955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,2,128,1,float16,fp8,0,0.12821867068608603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,2,128,1,fp8,fp8,0,0.09045867125193278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,4,128,1,float16,float16,0,0.11451199650764465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,8,128,1,float16,fp8,0,0.07343466579914093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,4,128,1,float16,fp8,0,0.11108799775441487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,8,4,128,1,fp8,fp8,0,0.10308800141016643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,1,128,1,float16,float16,0,0.055829331278800964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,8,128,1,float16,float16,0,0.07620800038178761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,1,128,1,float16,fp8,0,0.05518400172392527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,8,128,1,fp8,fp8,0,0.0641653339068095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,1,128,1,fp8,fp8,0,0.05026133358478546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,2,128,1,float16,float16,0,0.05691733459631602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,2,128,1,float16,fp8,0,0.056458666920661926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,2,128,1,fp8,fp8,0,0.05209066470464071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,4,128,1,float16,float16,0,0.07479999959468842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,4,128,1,fp8,fp8,0,0.0566240002711614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,8,4,128,1,float16,fp8,0,0.061466669042905174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,8,128,1,fp8,fp8,0,0.03827733298142751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,8,128,1,float16,fp8,0,0.038005332152048744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,1,128,1,float16,fp8,0,0.03385066737731298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,1,128,1,float16,float16,0,0.03346666693687439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,8,128,1,float16,float16,0,0.039733332892258964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,1,128,1,fp8,fp8,0,0.031231999397277832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,2,128,1,float16,float16,0,0.034245334565639496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,2,128,1,fp8,fp8,0,0.03123733401298523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,2,128,1,float16,fp8,0,0.03422400106986364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,8,128,1,float16,fp8,0,0.02462933212518692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,4,128,1,fp8,fp8,0,0.03381866713364919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,4,128,1,float16,float16,0,0.03551466763019562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,8,128,1,fp8,fp8,0,0.023061332603295643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,8,4,128,1,float16,fp8,0,0.03616533428430557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,8,128,1,float16,float16,0,0.024746666351954143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,1,128,1,float16,float16,0,0.022634667654832203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,2,128,1,float16,float16,0,0.023269332945346832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,1,128,1,fp8,fp8,0,0.021183999876181286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,2,128,1,fp8,fp8,0,0.021674667795499165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,2,128,1,float16,fp8,0,0.023408000667889912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,1,128,1,float16,fp8,0,0.022639999787012737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,4,128,1,float16,float16,0,0.02409599969784419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,4,128,1,fp8,fp8,0,0.02266666789849599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,8,4,128,1,float16,fp8,0,0.024512000381946564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,8,128,1,float16,float16,0,0.01926933353145917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,8,128,1,float16,fp8,0,0.018837332725524902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,1,128,1,float16,float16,0,0.01815466706951459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,8,128,1,fp8,fp8,0,0.019093333433071773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,1,128,1,fp8,fp8,0,0.018170667191346485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,1,128,1,float16,fp8,0,0.01838933303952217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,2,128,1,float16,float16,0,0.018346666047970455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,2,128,1,fp8,fp8,0,0.018650667121013004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,2,128,1,float16,fp8,0,0.01870399961868922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,4,128,1,float16,float16,0,0.01871466636657715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,4,128,1,float16,fp8,0,0.0186666672428449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,8,4,128,1,fp8,fp8,0,0.01921066641807556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,8,128,1,float16,float16,0,0.0173333336909612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,1,128,1,float16,float16,0,0.01693333312869072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,1,128,1,fp8,fp8,0,0.017103999853134155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,8,128,1,fp8,fp8,0,0.01747200017174085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,8,128,1,float16,fp8,0,0.017738666385412216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,1,128,1,float16,fp8,0,0.017103999853134155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,2,128,1,float16,float16,0,0.017071999609470367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,2,128,1,float16,fp8,0,0.017418666432301205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,4,128,1,float16,float16,0,0.017184000462293625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,4,128,1,float16,fp8,0,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,2,128,1,fp8,fp8,0,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,8,4,128,1,fp8,fp8,0,0.01754666616519292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,1,128,1,float16,float16,0,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,8,128,1,float16,fp8,0,0.016917333006858826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,8,128,1,fp8,fp8,0,0.016496000190575916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,8,128,1,float16,float16,0,0.016410666207472484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,1,128,1,float16,fp8,0,0.017018667111794155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,1,128,1,fp8,fp8,0,0.016901332885026932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,2,128,1,float16,float16,0,0.01637866720557213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,2,128,1,float16,fp8,0,0.016943999876578648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,2,128,1,fp8,fp8,0,0.016704000532627106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,4,128,1,float16,fp8,0,0.016864000509182613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,4,128,1,float16,float16,0,0.016645333419243496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,8,128,1,float16,float16,0,0.015919999529918034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,8,128,1,float16,fp8,0,0.016501333564519882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,8,128,1,fp8,fp8,0,0.016229332735141117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,1,128,1,float16,float16,0,0.01613333324591319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,8,4,128,1,fp8,fp8,0,0.01692266638080279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,1,128,1,float16,fp8,0,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,1,128,1,fp8,fp8,0,0.016303999970356624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,2,128,1,float16,float16,0,0.016261332978804905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,2,128,1,float16,fp8,0,0.016447999825080235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,2,128,1,fp8,fp8,0,0.016575999557971954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,4,128,1,float16,float16,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,4,128,1,float16,fp8,0,0.01674666628241539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,8,4,128,1,fp8,fp8,0,0.019205333044131596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,1,128,1,float16,float16,0,0.1111840009689331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,1,128,1,fp8,fp8,0,0.11406933267911275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,2,128,1,float16,float16,0,0.11537599563598633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,2,128,1,float16,fp8,0,0.14391466975212097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,1,128,1,float16,fp8,0,0.11086933811505635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,2,128,1,fp8,fp8,0,0.11776000261306763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,4,128,1,float16,float16,0,0.1253706713517507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,4,128,1,float16,fp8,0,0.12337600191434224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,1,128,1,float16,float16,0,0.060906668504079185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,8,128,1,float16,fp8,0,0.07294400036334991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,8,4,128,1,fp8,fp8,0,0.12764267126719156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,8,128,1,float16,float16,0,0.07519466678301494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,8,128,1,fp8,fp8,0,0.09282666444778442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,1,128,1,float16,fp8,0,0.0613919993241628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,1,128,1,fp8,fp8,0,0.06407999992370605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,2,128,1,float16,fp8,0,0.061792001128196716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,2,128,1,fp8,fp8,0,0.06735466420650482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,4,128,1,fp8,fp8,0,0.0796853353579839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,8,128,1,float16,float16,0,0.039477333426475525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,4,128,1,float16,fp8,0,0.06905599931875865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,4,128,1,float16,float16,0,0.06791999936103821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,8,2,128,1,float16,float16,0,0.062405332922935486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,8,128,1,float16,fp8,0,0.03867200016975403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,1,128,1,float16,float16,0,0.03562666724125544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,1,128,1,float16,fp8,0,0.03600533306598663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,8,128,1,fp8,fp8,0,0.044768000642458596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,2,128,1,float16,float16,0,0.036864000062147774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,1,128,1,fp8,fp8,0,0.037263999382654824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,2,128,1,fp8,fp8,0,0.038032000263532005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,2,128,1,float16,fp8,0,0.03628266602754593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,4,128,1,float16,fp8,0,0.037776000797748566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,4,128,1,float16,float16,0,0.03867200016975403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,8,4,128,1,fp8,fp8,0,0.04044266790151596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,8,128,1,float16,fp8,0,0.025146665672461193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,8,128,1,fp8,fp8,0,0.02625600000222524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,1,128,1,float16,fp8,0,0.02385066697994868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,8,128,1,float16,float16,0,0.02534399926662445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,1,128,1,fp8,fp8,0,0.024746666351954143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,2,128,1,float16,float16,0,0.024447999894618988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,1,128,1,float16,float16,0,0.023717333873112995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,2,128,1,float16,fp8,0,0.024271999796231587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,4,128,1,float16,float16,0,0.0252960001428922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,2,128,1,fp8,fp8,0,0.025125332176685333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,8,128,1,float16,float16,0,0.018960000326236088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,4,128,1,float16,fp8,0,0.025301332275072735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,8,4,128,1,fp8,fp8,0,0.02608533451954524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,8,128,1,float16,fp8,0,0.01911466692884763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,1,128,1,float16,float16,0,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,1,128,1,float16,fp8,0,0.017743999759356182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,1,128,1,fp8,fp8,0,0.018090666582187016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,2,128,1,float16,float16,0,0.017781333376963932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,2,128,1,float16,fp8,0,0.017850667238235474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,8,128,1,fp8,fp8,0,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,4,128,1,float16,fp8,0,0.01850133389234543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,4,128,1,float16,float16,0,0.01844800015290578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,4,128,1,fp8,fp8,0,0.019120000302791595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,8,128,1,float16,fp8,0,0.016154666741689045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,8,2,128,1,fp8,fp8,0,0.018320000420014065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,8,128,1,fp8,fp8,0,0.01718933383623759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,8,128,1,float16,float16,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,1,128,1,float16,float16,0,0.015813333292802174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,1,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,2,128,1,float16,fp8,0,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,1,128,1,fp8,fp8,0,0.01685333376129468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,2,128,1,float16,float16,0,0.015840000162522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,4,128,1,float16,float16,0,0.015962666521469753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,8,128,1,float16,fp8,0,0.015610666324694952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,8,128,1,float16,float16,0,0.015386667102575302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,4,128,1,fp8,fp8,0,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,2,128,1,fp8,fp8,0,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,1,128,1,float16,float16,0,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,8,128,1,fp8,fp8,0,0.016469333320856094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,1,128,1,float16,fp8,0,0.015824000040690105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,8,4,128,1,float16,fp8,0,0.016234666109085083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,1,128,1,fp8,fp8,0,0.01648533344268799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,2,128,1,float16,float16,0,0.015114666273196539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,2,128,1,float16,fp8,0,0.015941333025693893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,4,128,1,float16,float16,0,0.015279999623696009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,4,128,1,float16,fp8,0,0.015925332903862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,4,128,1,fp8,fp8,0,0.016586666305859882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,8,2,128,1,fp8,fp8,0,0.01639466608564059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,8,128,1,fp8,fp8,0,0.015882667154073715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,8,128,1,float16,fp8,0,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,1,128,1,float16,float16,0,0.015018666783968607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,2,128,1,float16,float16,0,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,1,128,1,fp8,fp8,0,0.016255999604860943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,8,128,1,float16,float16,0,0.014912000546852747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,2,128,1,fp8,fp8,0,0.016234666109085083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,4,128,1,fp8,fp8,0,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,4,128,1,float16,fp8,0,0.015557333827018738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,2,128,1,float16,fp8,0,0.01526933287580808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,8,128,1,float16,float16,0,0.014831999937693277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,4,128,1,float16,float16,0,0.014858666807413101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,8,128,1,fp8,fp8,0,0.01591466615597407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,8,1,128,1,float16,fp8,0,0.01552533358335495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,8,128,1,float16,fp8,0,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,1,128,1,float16,float16,0,0.014666666587193808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,2,128,1,fp8,fp8,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,1,128,1,float16,fp8,0,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,2,128,1,float16,float16,0,0.015008000036080679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,2,128,1,float16,fp8,0,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,4,128,1,float16,float16,0,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,1,128,1,fp8,fp8,0,0.015840000162522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,4,128,1,fp8,fp8,0,0.01600533351302147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,8,4,128,1,float16,fp8,0,0.015189333508412043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,1,128,1,float16,float16,0,0.07858133316040039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,2,128,1,float16,float16,0,0.07841066519419353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,2,128,1,float16,fp8,0,0.07816533247629802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,1,128,1,float16,fp8,0,0.07763200004895528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,1,128,1,fp8,fp8,0,0.09171199798583984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,4,128,1,float16,float16,0,0.08352532982826233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,2,128,1,fp8,fp8,0,0.0934399962425232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,8,128,1,float16,float16,0,0.04754666487375895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,4,128,1,fp8,fp8,0,0.10739200313886006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,1,128,1,float16,float16,0,0.042837331692377724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,8,128,1,float16,fp8,0,0.04584000011285146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,1,128,1,float16,fp8,0,0.042912001411120095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,8,4,128,1,float16,fp8,0,0.08426133791605632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,8,128,1,fp8,fp8,0,0.058965335289637245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,2,128,1,float16,fp8,0,0.04374399781227112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,2,128,1,float16,float16,0,0.04371733466784159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,1,128,1,fp8,fp8,0,0.05063466727733612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,2,128,1,fp8,fp8,0,0.05163733164469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,4,128,1,float16,fp8,0,0.044810667634010315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,4,128,1,float16,float16,0,0.04541866481304169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,8,4,128,1,fp8,fp8,0,0.05479466418425242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,8,128,1,float16,fp8,0,0.027888000011444092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,8,128,1,float16,float16,0,0.02770666778087616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,8,128,1,fp8,fp8,0,0.033376000821590424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,1,128,1,float16,fp8,0,0.027034667630990345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,1,128,1,fp8,fp8,0,0.03160000095764796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,2,128,1,fp8,fp8,0,0.03186133255561193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,2,128,1,float16,float16,0,0.027301333844661713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,4,128,1,float16,fp8,0,0.028618666032950085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,1,128,1,float16,float16,0,0.027210667729377747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,2,128,1,float16,fp8,0,0.027722666660944622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,4,128,1,float16,float16,0,0.028751999139785767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,8,128,1,float16,float16,0,0.020165332903464634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,8,4,128,1,fp8,fp8,0,0.033200000723203026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,1,128,1,float16,float16,0,0.019146667172511418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,8,128,1,float16,fp8,0,0.019738666713237762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,8,128,1,fp8,fp8,0,0.022543999056021374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,2,128,1,float16,float16,0,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,1,128,1,fp8,fp8,0,0.025093334416548412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,2,128,1,float16,fp8,0,0.019440000255902607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,1,128,1,float16,fp8,0,0.01918399954835574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,4,128,1,float16,float16,0,0.019850666324297588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,2,128,1,fp8,fp8,0,0.022170667846997578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,4,128,1,float16,fp8,0,0.019610666980346043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,8,4,128,1,fp8,fp8,0,0.022634667654832203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,1,128,1,float16,fp8,0,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,8,128,1,fp8,fp8,0,0.017749333133300144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,8,128,1,float16,float16,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,1,128,1,fp8,fp8,0,0.01716800034046173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,8,128,1,float16,fp8,0,0.01613866661985715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,2,128,1,float16,float16,0,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,2,128,1,float16,fp8,0,0.015594666202863058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,1,128,1,float16,float16,0,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,2,128,1,fp8,fp8,0,0.017407999684413273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,4,128,1,float16,float16,0,0.015557333827018738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,4,128,1,float16,fp8,0,0.016143999993801117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,8,128,1,float16,fp8,0,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,8,4,128,1,fp8,fp8,0,0.01783466711640358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,8,128,1,fp8,fp8,0,0.016549333930015564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,8,128,1,float16,float16,0,0.01481066644191742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,1,128,1,float16,fp8,0,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,1,128,1,fp8,fp8,0,0.016688000410795212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,2,128,1,float16,fp8,0,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,2,128,1,float16,float16,0,0.01482133318980535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,1,128,1,float16,float16,0,0.014581333845853806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,2,128,1,fp8,fp8,0,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,4,128,1,float16,float16,0,0.01462399959564209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,4,128,1,fp8,fp8,0,0.016656000167131424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,8,128,1,float16,fp8,0,0.014581333845853806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,8,128,1,float16,float16,0,0.014080000420411428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,8,128,1,fp8,fp8,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,8,4,128,1,float16,fp8,0,0.015018666783968607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,1,128,1,fp8,fp8,0,0.01616000011563301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,1,128,1,float16,fp8,0,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,2,128,1,float16,float16,0,0.014314666390419006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,1,128,1,float16,float16,0,0.014159999787807465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,2,128,1,float16,fp8,0,0.014853333433469137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,2,128,1,fp8,fp8,0,0.016186666985352833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,4,128,1,fp8,fp8,0,0.016303999970356624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,8,128,1,float16,float16,0,0.014101333916187286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,8,128,1,float16,fp8,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,4,128,1,float16,float16,0,0.014250667144854864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,1,128,1,float16,float16,0,0.014170666535695394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,8,4,128,1,float16,fp8,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,1,128,1,fp8,fp8,0,0.01609066625436147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,1,128,1,float16,fp8,0,0.0144213338692983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,2,128,1,float16,float16,0,0.014149333039919535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,8,128,1,fp8,fp8,0,0.015813333292802174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,2,128,1,float16,fp8,0,0.01463466634353002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,4,128,1,float16,float16,0,0.014032000054915747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,2,128,1,fp8,fp8,0,0.01626666635274887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,4,128,1,fp8,fp8,0,0.016186666985352833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,8,128,1,float16,float16,0,0.013562666873137156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,8,4,128,1,float16,fp8,0,0.014618666221698126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,1,128,1,float16,float16,0,0.013893333574136099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,8,128,1,float16,fp8,0,0.01394133393963178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,1,128,1,float16,fp8,0,0.0143306665122509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,1,128,1,fp8,fp8,0,0.016021333634853363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,8,128,1,fp8,fp8,0,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,2,128,1,float16,float16,0,0.013973332941532135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,2,128,1,float16,fp8,0,0.014432000617186228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,4,128,1,float16,float16,0,0.013850666582584381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,2,128,1,fp8,fp8,0,0.016186666985352833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,4,128,1,fp8,fp8,0,0.015935999651749928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,8,4,128,1,float16,fp8,0,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,1,128,1,float16,float16,0,0.06637866795063019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,2,128,1,float16,fp8,0,0.06711466610431671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,1,128,1,fp8,fp8,0,0.07799999912579854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,2,128,1,float16,float16,0,0.06640000144640605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,4,128,1,float16,float16,0,0.06950933237870534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,1,128,1,float16,fp8,0,0.06619200110435486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,2,128,1,fp8,fp8,0,0.07996800045172374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,4,128,1,float16,fp8,0,0.06778133412202199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,8,4,128,1,fp8,fp8,0,0.08392000198364258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,8,128,1,float16,float16,0,0.036389333506425224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,8,128,1,float16,fp8,0,0.03616533428430557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,8,128,1,fp8,fp8,0,0.04669866462548574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,1,128,1,float16,float16,0,0.03902400036652883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,2,128,1,fp8,fp8,0,0.04644800225893656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,2,128,1,float16,fp8,0,0.03928533444801966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,2,128,1,float16,float16,0,0.03887466589609782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,4,128,1,float16,float16,0,0.0397173340121905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,1,128,1,float16,fp8,0,0.03863999992609024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,4,128,1,fp8,fp8,0,0.0473333348830541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,4,128,1,float16,fp8,0,0.04020266731580099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,8,1,128,1,fp8,fp8,0,0.046112000942230225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,8,128,1,float16,float16,0,0.02422400067249934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,8,128,1,float16,fp8,0,0.024634666740894318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,8,128,1,fp8,fp8,0,0.029637334247430164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,1,128,1,float16,float16,0,0.02459733436505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,1,128,1,fp8,fp8,0,0.028901333610216778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,2,128,1,float16,float16,0,0.025061334172884624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,2,128,1,fp8,fp8,0,0.029285334050655365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,1,128,1,float16,fp8,0,0.0249439999461174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,2,128,1,float16,fp8,0,0.025072000920772552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,4,128,1,fp8,fp8,0,0.030031998952229817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,8,128,1,float16,float16,0,0.018197332819302876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,4,128,1,float16,fp8,0,0.02536533276240031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,8,128,1,float16,fp8,0,0.018378666291634243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,8,4,128,1,float16,float16,0,0.025455998877684276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,2,128,1,float16,float16,0,0.017829333742459614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,8,128,1,fp8,fp8,0,0.020938667158285778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,1,128,1,fp8,fp8,0,0.020506666352351505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,2,128,1,fp8,fp8,0,0.021183999876181286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,1,128,1,float16,fp8,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,2,128,1,float16,fp8,0,0.018613333503405254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,4,128,1,float16,fp8,0,0.01829333355029424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,1,128,1,float16,float16,0,0.01777600000301997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,4,128,1,fp8,fp8,0,0.021194666624069214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,8,4,128,1,float16,float16,0,0.01783466711640358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,1,128,1,float16,fp8,0,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,8,128,1,float16,fp8,0,0.01540800059835116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,1,128,1,float16,float16,0,0.014831999937693277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,8,128,1,float16,float16,0,0.015205333630243937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,2,128,1,float16,float16,0,0.014831999937693277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,8,128,1,fp8,fp8,0,0.01754666616519292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,2,128,1,float16,fp8,0,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,2,128,1,fp8,fp8,0,0.017231999586025875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,1,128,1,fp8,fp8,0,0.017029333859682083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,4,128,1,float16,float16,0,0.014959999670584997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,4,128,1,float16,fp8,0,0.015439999600251516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,8,128,1,float16,float16,0,0.014458666245142618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,8,4,128,1,fp8,fp8,0,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,1,128,1,float16,float16,0,0.014256000518798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,2,128,1,float16,fp8,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,8,128,1,fp8,fp8,0,0.01682666689157486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,1,128,1,fp8,fp8,0,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,2,128,1,float16,float16,0,0.01431999976436297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,8,128,1,float16,fp8,0,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,2,128,1,fp8,fp8,0,0.016442666451136272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,4,128,1,float16,float16,0,0.01423466702302297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,1,128,1,float16,fp8,0,0.014501333236694336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,4,128,1,float16,fp8,0,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,8,128,1,float16,float16,0,0.014165333161751429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,8,4,128,1,fp8,fp8,0,0.016762666404247284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,8,128,1,float16,fp8,0,0.014736000448465347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,1,128,1,float16,fp8,0,0.014698666830857595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,1,128,1,float16,float16,0,0.014080000420411428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,2,128,1,float16,float16,0,0.014282666146755219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,8,128,1,fp8,fp8,0,0.01617066686352094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,2,128,1,float16,fp8,0,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,4,128,1,float16,float16,0,0.014474666366974512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,4,128,1,float16,fp8,0,0.01479999969402949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,1,128,1,fp8,fp8,0,0.016127999871969223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,2,128,1,fp8,fp8,0,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,8,128,1,float16,float16,0,0.013679999858140945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,8,4,128,1,fp8,fp8,0,0.01632000009218852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,1,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,2,128,1,float16,fp8,0,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,8,128,1,fp8,fp8,0,0.015840000162522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,1,128,1,float16,float16,0,0.014170666535695394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,2,128,1,float16,float16,0,0.014186666657527288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,2,128,1,fp8,fp8,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,8,128,1,float16,fp8,0,0.014218666901191076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,4,128,1,float16,fp8,0,0.014442666123310724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,4,128,1,float16,float16,0,0.014096000542243322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,1,128,1,fp8,fp8,0,0.015919999529918034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,8,128,1,float16,float16,0,0.013376000026861826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,8,4,128,1,fp8,fp8,0,0.01603200038274129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,1,128,1,float16,fp8,0,0.014250667144854864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,8,128,1,float16,fp8,0,0.013776000589132309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,8,128,1,fp8,fp8,0,0.015706667055686314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,1,128,1,fp8,fp8,0,0.015850666910409927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,2,128,1,float16,fp8,0,0.014554666976133982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,2,128,1,fp8,fp8,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,1,128,1,float16,float16,0,0.014005333185195923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,4,128,1,float16,float16,0,0.013866666704416275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,4,128,1,fp8,fp8,0,0.015919999529918034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,2,128,1,float16,float16,0,0.013610667238632837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,8,4,128,1,float16,fp8,0,0.01434133326013883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,1,128,1,fp8,fp8,0,2.5936692555745444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,1,128,1,float16,float16,0,4.090885480244954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,1,128,1,float16,fp8,0,3.881162643432617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,4,128,1,float16,fp8,0,1.9651679992675781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,4,128,1,float16,float16,0,2.014234701792399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,2,128,1,float16,fp8,0,3.983824094136556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,2,128,1,fp8,fp8,0,2.6108959515889487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16384,4,2,128,1,float16,float16,0,3.911301294962565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,1,128,1,float16,float16,0,2.0623626708984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,1,128,1,float16,fp8,0,2.0199573834737143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,1,128,1,fp8,fp8,0,1.3632960319519043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,4,128,1,fp8,fp8,0,1.3236746788024902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,2,128,1,float16,fp8,0,2.028981367746989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,2,128,1,fp8,fp8,0,1.370181401570638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16384,4,2,128,1,float16,float16,0,2.065802733103434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,4,128,1,float16,float16,0,1.0671679973602295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,1,128,1,float16,float16,0,1.0937919616699219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,4,128,1,fp8,fp8,0,0.6553119818369547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,1,128,1,float16,fp8,0,1.0875306924184163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,4,128,1,float16,fp8,0,1.0754079818725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,1,128,1,fp8,fp8,0,0.6764427026112875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,2,128,1,float16,float16,0,1.0955359935760498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,1,128,1,float16,float16,0,0.506218671798706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,4,128,1,float16,float16,0,0.4933333396911621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,1,128,1,float16,fp8,0,0.5010613203048706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,2,128,1,float16,fp8,0,1.1052746772766113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16384,4,2,128,1,fp8,fp8,0,0.6830453077952067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,4,128,1,fp8,fp8,0,0.3457653522491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,4,128,1,float16,fp8,0,0.4917813142140706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,1,128,1,fp8,fp8,0,0.36187199751536053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,2,128,1,float16,float16,0,0.5100906689961752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,2,128,1,fp8,fp8,0,0.36427732308705646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16384,4,2,128,1,float16,fp8,0,0.5040213267008463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,1,128,1,fp8,fp8,0,1.5568480491638184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,1,128,1,float16,float16,0,2.348543961842855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,1,128,1,float16,fp8,0,2.2813013394673667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,4,128,1,float16,fp8,0,1.1786826451619465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,2,128,1,float16,fp8,0,2.285557270050049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,4,128,1,float16,float16,0,1.1924160321553547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,2,128,1,float16,float16,0,2.322277386983236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,12288,4,2,128,1,fp8,fp8,0,1.618794600168864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,1,128,1,float16,float16,0,1.2291253407796223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,4,128,1,fp8,fp8,0,0.7674773534138998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,1,128,1,fp8,fp8,0,0.7795039812723795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,1,128,1,float16,fp8,0,1.2039573192596436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,4,128,1,float16,float16,0,0.5835573275883993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,4,128,1,float16,fp8,0,0.5745973189671835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,2,128,1,float16,fp8,0,1.2132853666941326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,2,128,1,fp8,fp8,0,0.7978400389353434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,12288,4,2,128,1,float16,float16,0,1.2147626876831055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,1,128,1,float16,float16,0,0.572650671005249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,4,128,1,fp8,fp8,0,0.3986826737721761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,4,128,1,float16,float16,0,0.31706132491429645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,2,128,1,float16,fp8,0,0.5728106498718262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,1,128,1,float16,fp8,0,0.5681333144505819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,2,128,1,float16,float16,0,0.5843626658121744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,2,128,1,fp8,fp8,0,0.41598931948343915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,12288,4,1,128,1,fp8,fp8,0,0.41892798741658527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,4,128,1,float16,fp8,0,0.31709333260854083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,1,128,1,float16,float16,0,0.3261546691258748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,4,128,1,fp8,fp8,0,0.22963200012842813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,1,128,1,float16,fp8,0,0.3250880042711894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,1,128,1,fp8,fp8,0,0.23477866252263388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,2,128,1,float16,float16,0,0.32996267080307007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,2,128,1,float16,fp8,0,0.3277493317921956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,12288,4,2,128,1,fp8,fp8,0,0.24127999941507974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,1,128,1,float16,float16,0,1.6556159655253093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,1,128,1,float16,fp8,0,1.6350186665852864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,1,128,1,fp8,fp8,0,1.134895960489909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,2,128,1,float16,float16,0,1.6720213890075684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,4,128,1,float16,fp8,0,0.8902560075124105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,4,128,1,float16,float16,0,0.8683466911315918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,2,128,1,fp8,fp8,0,1.1488213539123535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,10240,4,2,128,1,float16,fp8,0,1.6527573267618816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,1,128,1,float16,float16,0,0.8804746468861898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,1,128,1,float16,fp8,0,0.8742453257242838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,4,128,1,fp8,fp8,0,0.563536008199056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,1,128,1,fp8,fp8,0,0.5685653289159139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,2,128,1,fp8,fp8,0,0.5778826475143433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,2,128,1,float16,float16,0,0.9114773273468018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,4,128,1,float16,float16,0,0.41762133439381915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,4,128,1,float16,fp8,0,0.4109119971593221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,1,128,1,float16,fp8,0,0.41976531346638996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,4,128,1,fp8,fp8,0,0.2999253273010254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,1,128,1,float16,float16,0,0.42417601744333905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,10240,4,2,128,1,float16,fp8,0,0.8838720321655273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,1,128,1,fp8,fp8,0,0.30608532826105755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,4,128,1,float16,fp8,0,0.23135999838511148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,4,128,1,float16,float16,0,0.2336639960606893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,2,128,1,fp8,fp8,0,0.30939199527104694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,2,128,1,float16,float16,0,0.43486932913462323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,4,128,1,fp8,fp8,0,0.17302932341893515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,10240,4,2,128,1,float16,fp8,0,0.4233280022939046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,1,128,1,float16,float16,0,0.2381440003712972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,2,128,1,float16,float16,0,0.2400746742884318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,1,128,1,fp8,fp8,0,0.1716960072517395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,2,128,1,float16,fp8,0,0.23865600426991782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,2,128,1,fp8,fp8,0,0.17826666434605917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,10240,4,1,128,1,float16,fp8,0,0.23755200703938803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,1,128,1,float16,float16,0,2.1499733924865723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,1,128,1,fp8,fp8,0,1.4856053988138835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,1,128,1,float16,fp8,0,2.127072016398112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,4,128,1,float16,float16,0,1.1767466862996419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,2,128,1,float16,float16,0,2.175429344177246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,2,128,1,fp8,fp8,0,1.5027947425842285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,8192,4,2,128,1,float16,fp8,0,2.1405280431111655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,4,128,1,float16,fp8,0,1.1167999903361003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,1,128,1,float16,float16,0,1.1217866738637288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,1,128,1,float16,fp8,0,1.1079359849294026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,4,128,1,fp8,fp8,0,0.7773226896921793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,1,128,1,fp8,fp8,0,0.8126239776611328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,2,128,1,float16,float16,0,1.129578669865926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,2,128,1,float16,fp8,0,1.114303986231486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,8192,4,2,128,1,fp8,fp8,0,0.790234645207723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,1,128,1,float16,float16,0,0.6053173144658407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,4,128,1,float16,float16,0,0.6022773186365763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,4,128,1,float16,fp8,0,0.5924959977467855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,1,128,1,float16,fp8,0,0.6072479883829752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,1,128,1,fp8,fp8,0,0.39106134573618573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,4,128,1,fp8,fp8,0,0.3930186827977498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,2,128,1,float16,float16,0,0.6071039835611979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,2,128,1,float16,fp8,0,0.6045493284861246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,4,128,1,float16,fp8,0,0.27902400493621826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,1,128,1,float16,float16,0,0.2823093334833781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,4,128,1,fp8,fp8,0,0.21817066272099814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,8192,4,2,128,1,fp8,fp8,0,0.3985813458760579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,4,128,1,float16,float16,0,0.2843946615854899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,1,128,1,fp8,fp8,0,0.21184533834457397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,1,128,1,float16,fp8,0,0.28062933683395386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,2,128,1,float16,fp8,0,0.2834879954655965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,2,128,1,float16,float16,0,0.2853493293126424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,4,128,1,float16,fp8,0,0.15235199530919394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,8192,4,2,128,1,fp8,fp8,0,0.21260799964269003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,4,128,1,float16,float16,0,0.1535360018412272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,4,128,1,fp8,fp8,0,0.1165706713994344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,1,128,1,float16,float16,0,0.1527359982331594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,1,128,1,fp8,fp8,0,0.11427733302116394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,2,128,1,float16,float16,0,0.15524267156918845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,2,128,1,fp8,fp8,0,0.1162559986114502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,2,128,1,float16,fp8,0,0.15356799960136414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,8192,4,1,128,1,float16,fp8,0,0.1527253290017446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,1,128,1,float16,fp8,0,1.3027466932932537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,1,128,1,float16,float16,0,1.3808959325154622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,1,128,1,fp8,fp8,0,0.9233012994130453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,4,128,1,float16,float16,0,0.6846026579538981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,2,128,1,fp8,fp8,0,0.9380746682484945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,2,128,1,float16,fp8,0,1.2985706329345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,4,128,1,float16,fp8,0,0.6759146849314371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,6144,4,2,128,1,float16,float16,0,1.3233493169148762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,1,128,1,float16,float16,0,0.7156266371409098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,1,128,1,float16,fp8,0,0.6751680374145508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,4,128,1,fp8,fp8,0,0.48285333315531415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,1,128,1,fp8,fp8,0,0.4670666853586833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,2,128,1,float16,float16,0,0.6938506762186686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,2,128,1,float16,fp8,0,0.6834666728973389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,6144,4,2,128,1,fp8,fp8,0,0.4807200034459432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,4,128,1,fp8,fp8,0,0.25154133637746173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,4,128,1,float16,float16,0,0.3356959819793701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,1,128,1,float16,float16,0,0.32982399066289264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,4,128,1,float16,fp8,0,0.34940799077351886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,1,128,1,float16,fp8,0,0.3242400089899699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,1,128,1,fp8,fp8,0,0.24897066752115884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,2,128,1,float16,float16,0,0.3341439962387085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,2,128,1,float16,fp8,0,0.33035733302434284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,4,128,1,float16,fp8,0,0.18482667207717896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,4,128,1,float16,float16,0,0.18594666322072348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,1,128,1,float16,float16,0,0.18635733922322592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,6144,4,2,128,1,fp8,fp8,0,0.2518613338470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,1,128,1,float16,fp8,0,0.18409067392349243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,4,128,1,fp8,fp8,0,0.14819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,1,128,1,fp8,fp8,0,0.13863466183344522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,4,128,1,float16,float16,0,0.11356799801190694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,4,128,1,fp8,fp8,0,0.0886293351650238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,4,128,1,float16,fp8,0,0.11331733067830403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,2,128,1,fp8,fp8,0,0.144186665614446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,1,128,1,float16,float16,0,0.11577600240707397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,2,128,1,float16,float16,0,0.1890986760457357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,6144,4,2,128,1,float16,fp8,0,0.18739734093348184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,1,128,1,float16,fp8,0,0.11524266997973125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,2,128,1,float16,float16,0,0.11614400148391724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,1,128,1,fp8,fp8,0,0.08684266606966655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,2,128,1,fp8,fp8,0,0.08881066242853801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,6144,4,2,128,1,float16,fp8,0,0.11563733220100403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,1,128,1,float16,float16,0,1.4278346697489421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,1,128,1,fp8,fp8,0,0.9542187054951986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,1,128,1,float16,fp8,0,1.2792426745096843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,4,128,1,float16,float16,0,0.6817333698272705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,4,128,1,float16,fp8,0,0.661301334698995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,2,128,1,float16,float16,0,1.3326133092244465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,2,128,1,fp8,fp8,0,0.9721759955088297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,4096,4,2,128,1,float16,fp8,0,1.2981173197428386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,1,128,1,float16,float16,0,0.6767573356628418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,1,128,1,float16,fp8,0,0.6584266821543375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,1,128,1,fp8,fp8,0,0.49819199244181317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,2,128,1,float16,float16,0,0.6795573234558105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,4,128,1,fp8,fp8,0,0.557642658551534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,4,128,1,float16,float16,0,0.3641066551208496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,2,128,1,float16,fp8,0,0.6665653387705485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,4096,4,2,128,1,fp8,fp8,0,0.505295991897583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,4,128,1,float16,fp8,0,0.36087465286254883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,1,128,1,float16,fp8,0,0.35148799419403076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,1,128,1,fp8,fp8,0,0.26419732968012494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,1,128,1,float16,float16,0,0.3605813185373942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,2,128,1,float16,float16,0,0.3639306624730428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,2,128,1,float16,fp8,0,0.35760533809661865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,4,128,1,fp8,fp8,0,0.2625280022621155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,4096,4,2,128,1,fp8,fp8,0,0.25675733884175617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,4,128,1,float16,float16,0,0.17804799477259317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,4,128,1,float16,fp8,0,0.17347200711568198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,1,128,1,float16,fp8,0,0.169429341952006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,1,128,1,fp8,fp8,0,0.1374613344669342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,4,128,1,fp8,fp8,0,0.15033066272735596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,1,128,1,float16,float16,0,0.17223999897638956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,2,128,1,float16,fp8,0,0.17166932423909506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,2,128,1,fp8,fp8,0,0.13843199610710144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,4096,4,2,128,1,float16,float16,0,0.17537067333857217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,4,128,1,float16,fp8,0,0.09491200248400371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,4,128,1,float16,float16,0,0.09498133261998494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,1,128,1,float16,float16,0,0.09313066800435384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,4,128,1,fp8,fp8,0,0.0798933357000351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,1,128,1,float16,fp8,0,0.09258133172988892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,1,128,1,fp8,fp8,0,0.07341866691907246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,2,128,1,fp8,fp8,0,0.07597866654396057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,4,128,1,float16,float16,0,0.07915199796358745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,4,128,1,float16,fp8,0,0.07901866734027863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,4,128,1,fp8,fp8,0,0.06192000210285187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,2,128,1,float16,float16,0,0.09405333797136943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,4096,4,2,128,1,float16,fp8,0,0.09318932890892029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,1,128,1,float16,fp8,0,0.08108266691366832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,1,128,1,float16,float16,0,0.08213866750399272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,1,128,1,fp8,fp8,0,0.06244266529877981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,2,128,1,float16,float16,0,0.08169066905975342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,2,128,1,float16,fp8,0,0.08155733346939087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,4096,4,2,128,1,fp8,fp8,0,0.06267199913660686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,1,128,1,float16,float16,0,0.8221866289774576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,1,128,1,fp8,fp8,0,0.6134506861368815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,1,128,1,float16,fp8,0,0.7938026587168375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,4,128,1,float16,float16,0,0.4880319833755493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,2,128,1,float16,float16,0,0.8353119691212972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,4,128,1,float16,fp8,0,0.4251573483149211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,2,128,1,float16,fp8,0,0.8072160085042318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,3072,4,2,128,1,fp8,fp8,0,0.6275946696599325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,1,128,1,float16,float16,0,0.4256906509399414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,1,128,1,float16,fp8,0,0.4464799960454305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,4,128,1,fp8,fp8,0,0.34124799569447833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,2,128,1,float16,float16,0,0.4339573383331299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,4,128,1,float16,float16,0,0.2191093365351359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,2,128,1,float16,fp8,0,0.4224373499552409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,1,128,1,fp8,fp8,0,0.3124053279558818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,3072,4,2,128,1,fp8,fp8,0,0.3240426580111186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,4,128,1,float16,fp8,0,0.2163146734237671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,1,128,1,float16,fp8,0,0.2048906683921814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,1,128,1,fp8,fp8,0,0.16551466782887778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,4,128,1,fp8,fp8,0,0.19270400206247965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,1,128,1,float16,float16,0,0.20858667294184366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,2,128,1,float16,float16,0,0.21400533119837442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,2,128,1,fp8,fp8,0,0.17001599073410034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,3072,4,2,128,1,float16,fp8,0,0.21102933088938394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,4,128,1,float16,float16,0,0.11868266264597575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,1,128,1,float16,float16,0,0.11576533317565918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,1,128,1,fp8,fp8,0,0.09101333220799764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,1,128,1,float16,fp8,0,0.11410666505495708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,2,128,1,float16,float16,0,0.11769066254297893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,4,128,1,fp8,fp8,0,0.09975999593734741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,4,128,1,float16,fp8,0,0.11780266960461934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,2,128,1,fp8,fp8,0,0.09642133116722107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,3072,4,2,128,1,float16,fp8,0,0.11629333098729451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,1,128,1,float16,float16,0,0.06948799888292949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,4,128,1,fp8,fp8,0,0.059418668349583946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,4,128,1,float16,fp8,0,0.06905066470305125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,4,128,1,float16,float16,0,0.06965333223342896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,1,128,1,float16,fp8,0,0.06862399975458781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,1,128,1,fp8,fp8,0,0.05558399856090546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,2,128,1,float16,float16,0,0.0698880006869634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,2,128,1,fp8,fp8,0,0.05624533196290334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,1,128,1,float16,float16,0,0.06438399851322174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,4,128,1,float16,float16,0,0.06243200103441874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,4,128,1,fp8,fp8,0,0.049584001302719116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,4,128,1,float16,fp8,0,0.062463998794555664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,1,128,1,float16,fp8,0,0.06408533453941345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,1,128,1,fp8,fp8,0,0.04984533290068308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,3072,4,2,128,1,float16,fp8,0,0.06949333349863689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,2,128,1,float16,fp8,0,0.06402666866779327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,2,128,1,fp8,fp8,0,0.050197333097457886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,3072,4,2,128,1,float16,float16,0,0.06457599997520447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,1,128,1,float16,float16,0,0.9013226826985677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,1,128,1,fp8,fp8,0,0.7138933340708414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,2,128,1,float16,float16,0,0.9216693242390951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,1,128,1,float16,fp8,0,0.935370683670044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,4,128,1,float16,float16,0,0.4870773156483968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,4,128,1,float16,fp8,0,0.45375998814900714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,2,128,1,fp8,fp8,0,0.7273759841918945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,2048,4,2,128,1,float16,fp8,0,0.8558826446533203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,1,128,1,float16,float16,0,0.46425068378448486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,1,128,1,float16,fp8,0,0.43754132588704425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,1,128,1,fp8,fp8,0,0.40403199195861816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,2,128,1,float16,float16,0,0.4743359883626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,2,128,1,float16,fp8,0,0.4434933265050252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,2,128,1,fp8,fp8,0,0.3763360182444255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,2048,4,4,128,1,fp8,fp8,0,0.39138134320576984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,4,128,1,float16,float16,0,0.25685866673787433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,4,128,1,float16,fp8,0,0.24342399835586548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,1,128,1,fp8,fp8,0,0.1851200064023336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,1,128,1,float16,float16,0,0.2587146759033203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,1,128,1,float16,fp8,0,0.23185600837071738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,4,128,1,fp8,fp8,0,0.20202666521072388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,2,128,1,float16,float16,0,0.2453493277231852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,2,128,1,float16,fp8,0,0.23710399866104126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,4,128,1,float16,fp8,0,0.12363200386365254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,1,128,1,float16,float16,0,0.11830400427182515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,1,128,1,float16,fp8,0,0.11468266447385152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,4,128,1,float16,float16,0,0.12812800208727518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,2048,4,2,128,1,fp8,fp8,0,0.1911733349164327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,4,128,1,fp8,fp8,0,0.12097066640853882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,1,128,1,fp8,fp8,0,0.10072533289591472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,2,128,1,float16,float16,0,0.12069333593050639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,2,128,1,float16,fp8,0,0.11730666955312093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,2048,4,2,128,1,fp8,fp8,0,0.10356799761454265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,4,128,1,float16,float16,0,0.06835733354091644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,4,128,1,float16,fp8,0,0.06742933392524719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,1,128,1,fp8,fp8,0,0.05331199864546458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,1,128,1,float16,fp8,0,0.06229866544405619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,2,128,1,float16,float16,0,0.06513600051403046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,4,128,1,fp8,fp8,0,0.06201600035031637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,2,128,1,float16,fp8,0,0.06348800162474315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,1,128,1,float16,float16,0,0.06379200021425883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,2048,4,2,128,1,fp8,fp8,0,0.05619200070699056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,4,128,1,fp8,fp8,0,0.04251199960708618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,4,128,1,float16,fp8,0,0.05012266834576925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,1,128,1,float16,float16,0,0.051034669081370033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,1,128,1,fp8,fp8,0,0.04159999887148539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,4,128,1,float16,float16,0,0.0503359983364741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,2,128,1,float16,fp8,0,0.050981332858403526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,1,128,1,float16,fp8,0,0.0503359983364741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,4,128,1,float16,fp8,0,0.04597333570321401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,4,128,1,float16,float16,0,0.04614933331807455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,2,128,1,fp8,fp8,0,0.04200533529122671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,2048,4,2,128,1,float16,float16,0,0.051311999559402466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,4,128,1,fp8,fp8,0,0.037802666425704956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,1,128,1,float16,fp8,0,0.047423998514811196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,2,128,1,float16,float16,0,0.047695999344189964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,1,128,1,float16,float16,0,0.047775998711586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,2,128,1,float16,fp8,0,0.04736533264319102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,2,128,1,fp8,fp8,0,0.037903999288876854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,2048,4,1,128,1,fp8,fp8,0,0.0376800000667572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,1,128,1,float16,fp8,0,0.5350826581319174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,1,128,1,fp8,fp8,0,0.49409600098927814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,1,128,1,float16,float16,0,0.6737173398335775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,4,128,1,float16,float16,0,0.33079467217127484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,2,128,1,float16,float16,0,0.6172693173090616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,2,128,1,fp8,fp8,0,0.5061706701914469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1536,4,2,128,1,float16,fp8,0,0.5491146643956503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,4,128,1,float16,fp8,0,0.30317866802215576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,1,128,1,float16,float16,0,0.31439467271169025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,1,128,1,fp8,fp8,0,0.24809600909550986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,4,128,1,fp8,fp8,0,0.28996266921361286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,1,128,1,float16,fp8,0,0.3119093378384908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,2,128,1,float16,fp8,0,0.29183467229207355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,2,128,1,float16,float16,0,0.32283200820287067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,4,128,1,float16,float16,0,0.17491199572881064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,1,128,1,float16,float16,0,0.15362667044003805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,4,128,1,float16,fp8,0,0.16530666748682657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1536,4,2,128,1,fp8,fp8,0,0.2601013382275899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,4,128,1,fp8,fp8,0,0.14328533411026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,1,128,1,float16,fp8,0,0.15177067120869955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,1,128,1,fp8,fp8,0,0.13179199894269308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,2,128,1,float16,float16,0,0.1602773368358612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,4,128,1,float16,float16,0,0.08922666311264038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,2,128,1,fp8,fp8,0,0.13547199964523315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,4,128,1,fp8,fp8,0,0.08026666442553203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,1,128,1,fp8,fp8,0,0.07089599967002869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,1,128,1,float16,float16,0,0.08202666540940602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,1,128,1,float16,fp8,0,0.07823466757933299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,4,128,1,float16,fp8,0,0.08669867118199666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1536,4,2,128,1,float16,fp8,0,0.1511360009511312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,2,128,1,float16,fp8,0,0.08197333415349324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,2,128,1,float16,float16,0,0.0848586658636729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,4,128,1,float16,float16,0,0.04801600178082784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1536,4,2,128,1,fp8,fp8,0,0.07645333309968312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,4,128,1,fp8,fp8,0,0.04655466477076212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,1,128,1,float16,float16,0,0.04761599997679392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,1,128,1,float16,fp8,0,0.0462773342927297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,4,128,1,float16,fp8,0,0.04869333406289419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,1,128,1,fp8,fp8,0,0.04043733328580856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,2,128,1,float16,fp8,0,0.04717333118120829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,2,128,1,fp8,fp8,0,0.0415786678592364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1536,4,2,128,1,float16,float16,0,0.047781333327293396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,4,128,1,float16,float16,0,0.04113066693147024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,1,128,1,float16,fp8,0,0.041333332657814026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,4,128,1,fp8,fp8,0,0.0353973334034284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,4,128,1,float16,fp8,0,0.041189332803090416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,1,128,1,fp8,fp8,0,0.034389334420363106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,2,128,1,float16,float16,0,0.042021334171295166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,1,128,1,float16,float16,0,0.041840001940727234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,2,128,1,float16,fp8,0,0.041802664597829185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1536,4,2,128,1,fp8,fp8,0,0.03499733408292135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,4,128,1,float16,float16,0,0.03794133414824804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,4,128,1,float16,fp8,0,0.03864533454179764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,1,128,1,float16,float16,0,0.03925866633653641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,2,128,1,float16,float16,0,0.039450667798519135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,2,128,1,float16,fp8,0,0.03902933249870936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,1,128,1,float16,fp8,0,0.03897066662708918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,1,128,1,fp8,fp8,0,0.03209066639343897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,2,128,1,fp8,fp8,0,0.03209066639343897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1536,4,4,128,1,fp8,fp8,0,0.03182400017976761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,1,128,1,float16,fp8,0,0.5360053380330404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,1,128,1,float16,float16,0,0.5421280066172282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,1,128,1,fp8,fp8,0,0.4416266679763794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,4,128,1,float16,float16,0,0.32762134075164795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,2,128,1,float16,fp8,0,0.555951992670695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,4,128,1,float16,fp8,0,0.3156053423881531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,2,128,1,float16,float16,0,0.5676693518956503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,1024,4,2,128,1,fp8,fp8,0,0.5229119857152303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,1,128,1,float16,fp8,0,0.280074675877889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,1,128,1,fp8,fp8,0,0.23122133811314902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,1,128,1,float16,float16,0,0.28217599789301556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,4,128,1,fp8,fp8,0,0.26949334144592285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,2,128,1,float16,float16,0,0.2945493261019389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,2,128,1,fp8,fp8,0,0.24222399791081747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,4,128,1,float16,float16,0,0.22127467393875122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,1024,4,2,128,1,float16,fp8,0,0.2905760010083516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,4,128,1,fp8,fp8,0,0.13924800356229147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,4,128,1,float16,fp8,0,0.1696000099182129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,1,128,1,float16,fp8,0,0.15246400237083435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,1,128,1,float16,float16,0,0.15416000286738077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,1,128,1,fp8,fp8,0,0.12010133266448975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,2,128,1,float16,fp8,0,0.18509334325790405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,2,128,1,float16,float16,0,0.16129600008328757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,1024,4,2,128,1,fp8,fp8,0,0.12609066565831503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,4,128,1,float16,float16,0,0.09477866689364116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,4,128,1,float16,fp8,0,0.092357337474823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,1,128,1,float16,fp8,0,0.07875733574231465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,4,128,1,fp8,fp8,0,0.07733333110809326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,1,128,1,float16,float16,0,0.07948266466458638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,1,128,1,fp8,fp8,0,0.0721013347307841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,2,128,1,float16,float16,0,0.08288000027338664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,4,128,1,float16,float16,0,0.05023466547330221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,2,128,1,fp8,fp8,0,0.06996800005435944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,4,128,1,fp8,fp8,0,0.04534400006135305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,4,128,1,float16,fp8,0,0.047930667797724404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,1024,4,2,128,1,float16,fp8,0,0.08175999919573466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,1,128,1,float16,float16,0,0.04364266494909922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,1,128,1,float16,fp8,0,0.04345599810282389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,1,128,1,fp8,fp8,0,0.037658666570981346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,2,128,1,float16,float16,0,0.04534400006135305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,2,128,1,float16,fp8,0,0.04523199796676636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,1024,4,2,128,1,fp8,fp8,0,0.03961066653331121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,4,128,1,float16,float16,0,0.033370666205883026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,1,128,1,float16,float16,0,0.032069332897663116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,1,128,1,fp8,fp8,0,0.0276853342851003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,4,128,1,float16,fp8,0,0.033439998825391136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,1,128,1,float16,fp8,0,0.032357332607110344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,4,128,1,fp8,fp8,0,0.028959999481836956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,2,128,1,float16,float16,0,0.03262399882078171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,2,128,1,float16,fp8,0,0.03272533416748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,1024,4,2,128,1,fp8,fp8,0,0.02815466622511546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,4,128,1,float16,float16,0,0.029461334149042766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,2,128,1,float16,float16,0,0.029637334247430164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,4,128,1,float16,fp8,0,0.02959999938805898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,1,128,1,float16,float16,0,0.029530666768550873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,2,128,1,float16,fp8,0,0.029802667597929638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,2,128,1,fp8,fp8,0,0.024773334463437397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,1,128,1,float16,fp8,0,0.02951466788848241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,4,128,1,fp8,fp8,0,0.025279998779296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,1024,4,1,128,1,fp8,fp8,0,0.02492266645034154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,4,128,1,float16,float16,0,0.02769600103298823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,4,128,1,float16,fp8,0,0.028160000840822857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,4,128,1,fp8,fp8,0,0.02309866746266683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,1,128,1,float16,fp8,0,0.028698667883872986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,1,128,1,float16,float16,0,0.028421332438786823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,2,128,1,fp8,fp8,0,0.023818666736284893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,2,128,1,float16,float16,0,0.028325334191322327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,1,128,1,fp8,fp8,0,0.023573334018389385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,1024,4,2,128,1,float16,fp8,0,0.02884799987077713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,1,128,1,float16,float16,0,0.4339306751887004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,1,128,1,float16,fp8,0,0.4286666711171468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,1,128,1,fp8,fp8,0,0.40725334485371906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,2,128,1,float16,fp8,0,0.45187199115753174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,2,128,1,float16,float16,0,0.46064531803131104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,4,128,1,float16,fp8,0,0.26574933528900146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,4,128,1,float16,float16,0,0.2754879991213481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,512,4,2,128,1,fp8,fp8,0,0.399674654006958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,1,128,1,float16,float16,0,0.22639999787012735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,1,128,1,float16,fp8,0,0.2632693250974019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,1,128,1,fp8,fp8,0,0.19576533635457358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,4,128,1,fp8,fp8,0,0.23404266436894736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,2,128,1,float16,float16,0,0.23947733640670776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,2,128,1,float16,fp8,0,0.23593600591023764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,4,128,1,float16,fp8,0,0.14381866653760275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,1,128,1,float16,float16,0,0.12351466218630473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,512,4,2,128,1,fp8,fp8,0,0.20432533820470175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,1,128,1,float16,fp8,0,0.12201066811879475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,4,128,1,float16,float16,0,0.14923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,1,128,1,fp8,fp8,0,0.10207999746004741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,4,128,1,fp8,fp8,0,0.167194664478302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,2,128,1,float16,float16,0,0.13064533472061157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,4,128,1,float16,fp8,0,0.08221333225568135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,2,128,1,fp8,fp8,0,0.10605333248774211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,512,4,2,128,1,float16,fp8,0,0.13029332955678305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,1,128,1,float16,float16,0,0.06677333513895671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,4,128,1,fp8,fp8,0,0.06876799960931142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,1,128,1,fp8,fp8,0,0.057914664347966514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,4,128,1,float16,float16,0,0.08546132842699687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,1,128,1,float16,fp8,0,0.06969599922498067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,2,128,1,float16,float16,0,0.07076799869537354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,4,128,1,float16,float16,0,0.04304533203442892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,2,128,1,float16,fp8,0,0.06919999917348225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,512,4,2,128,1,fp8,fp8,0,0.06112533311049143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,4,128,1,fp8,fp8,0,0.04027199993530909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,4,128,1,float16,fp8,0,0.04257066547870636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,1,128,1,float16,float16,0,0.03783999880154928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,1,128,1,fp8,fp8,0,0.03327466547489166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,2,128,1,float16,fp8,0,0.0386559988061587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,2,128,1,float16,float16,0,0.039066667358080544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,2,128,1,fp8,fp8,0,0.034927998979886375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,512,4,1,128,1,float16,fp8,0,0.03757333258787791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,4,128,1,float16,float16,0,0.027056001126766205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,4,128,1,float16,fp8,0,0.027903998891512554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,4,128,1,fp8,fp8,0,0.025114665428797405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,1,128,1,float16,float16,0,0.02606933315594991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,1,128,1,float16,fp8,0,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,1,128,1,fp8,fp8,0,0.023525332411130268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,2,128,1,float16,float16,0,0.026362667481104534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,2,128,1,fp8,fp8,0,0.024277334411938984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,512,4,2,128,1,float16,fp8,0,0.025834667185942333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,4,128,1,float16,float16,0,0.0239680012067159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,4,128,1,fp8,fp8,0,0.02128533273935318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,1,128,1,float16,float16,0,0.022624000906944275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,4,128,1,float16,fp8,0,0.02348266790310542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,1,128,1,float16,fp8,0,0.02260799954334895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,1,128,1,fp8,fp8,0,0.02062400057911873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,4,128,1,float16,fp8,0,0.021551998953024547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,2,128,1,fp8,fp8,0,0.020879998803138733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,2,128,1,float16,float16,0,0.02298133323589961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,4,128,1,float16,float16,0,0.021551998953024547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,4,128,1,fp8,fp8,0,0.01941866676012675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,1,128,1,float16,float16,0,0.02163733293612798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,512,4,2,128,1,float16,fp8,0,0.02309866746266683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,1,128,1,float16,fp8,0,0.021856000026067097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,1,128,1,fp8,fp8,0,0.01897066707412402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,2,128,1,float16,float16,0,0.021562665700912476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,2,128,1,fp8,fp8,0,0.019776000330845516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,512,4,2,128,1,float16,fp8,0,0.0220266655087471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,4,128,1,float16,fp8,0,0.02111999938885371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,1,128,1,float16,fp8,0,0.021583999196688335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,1,128,1,float16,float16,0,0.021189334491888683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,4,128,1,float16,float16,0,0.02065066620707512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,2,128,1,float16,float16,0,0.021194666624069214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,2,128,1,float16,fp8,0,0.021573332448800404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,2,128,1,fp8,fp8,0,0.018789333601792652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,4,128,1,fp8,fp8,0,0.018559999763965607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,512,4,1,128,1,fp8,fp8,0,0.019050666441520054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,1,128,1,float16,fp8,0,0.1928106745084127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,1,128,1,float16,float16,0,0.1946559945742289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,1,128,1,fp8,fp8,0,0.17071467638015747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,2,128,1,float16,float16,0,0.21158399184544882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,2,128,1,float16,fp8,0,0.2074026664098104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,4,128,1,float16,fp8,0,0.13476799925168356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,256,4,2,128,1,fp8,fp8,0,0.1819146672884623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,4,128,1,float16,float16,0,0.2047626574834188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,1,128,1,float16,float16,0,0.10780266920725505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,4,128,1,fp8,fp8,0,0.11769599715868632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,1,128,1,float16,fp8,0,0.10735467076301575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,1,128,1,fp8,fp8,0,0.09278933207194011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,2,128,1,fp8,fp8,0,0.09784533580144246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,2,128,1,float16,float16,0,0.11512533823649089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,256,4,2,128,1,float16,fp8,0,0.11365333199501038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,4,128,1,float16,fp8,0,0.07796266674995422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,1,128,1,float16,float16,0,0.05995733539263407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,4,128,1,float16,float16,0,0.11023466785748799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,1,128,1,float16,fp8,0,0.059445331494013466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,4,128,1,fp8,fp8,0,0.06538666784763336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,1,128,1,fp8,fp8,0,0.05239999790986379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,2,128,1,float16,float16,0,0.0639519989490509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,1,128,1,float16,float16,0,0.03552533437808355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,4,128,1,float16,float16,0,0.04062400013208389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,2,128,1,fp8,fp8,0,0.05692266424496969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,1,128,1,float16,fp8,0,0.035258665680885315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,4,128,1,fp8,fp8,0,0.03869866579771042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,4,128,1,float16,fp8,0,0.041749333341916404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,256,4,2,128,1,float16,fp8,0,0.06355200211207072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,1,128,1,fp8,fp8,0,0.030767999589443207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,2,128,1,float16,float16,0,0.03670933345953623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,2,128,1,float16,fp8,0,0.036133334040641785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,256,4,2,128,1,fp8,fp8,0,0.03254933406909307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,4,128,1,float16,fp8,0,0.025626666843891144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,4,128,1,float16,float16,0,0.025621332228183746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,1,128,1,float16,fp8,0,0.023893333971500397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,1,128,1,float16,float16,0,0.02402666707833608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,4,128,1,fp8,fp8,0,0.02316266546646754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,1,128,1,fp8,fp8,0,0.021759999295075733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,2,128,1,float16,float16,0,0.024245334168275196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,2,128,1,float16,fp8,0,0.024447999894618988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,256,4,2,128,1,fp8,fp8,0,0.02229333420594533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,4,128,1,float16,float16,0,0.02042666698495547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,4,128,1,float16,fp8,0,0.020629333953062694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,1,128,1,float16,fp8,0,0.019472000499566395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,1,128,1,fp8,fp8,0,0.018624000251293182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,2,128,1,float16,float16,0,0.01972266659140587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,4,128,1,fp8,fp8,0,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,2,128,1,float16,fp8,0,0.019776000330845516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,2,128,1,fp8,fp8,0,0.018757333358128864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,256,4,1,128,1,float16,float16,0,0.01942933350801468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,4,128,1,float16,float16,0,0.018288000176350277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,4,128,1,float16,fp8,0,0.0183999997874101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,4,128,1,fp8,fp8,0,0.017573333034912746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,1,128,1,float16,float16,0,0.018197332819302876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,2,128,1,float16,float16,0,0.01841066653529803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,1,128,1,float16,fp8,0,0.01871466636657715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,1,128,1,fp8,fp8,0,0.017269333203633625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,4,128,1,float16,float16,0,0.017717332889636356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,4,128,1,float16,fp8,0,0.018144000321626663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,2,128,1,float16,fp8,0,0.01855466639002164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,4,128,1,fp8,fp8,0,0.016634666671355564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,256,4,2,128,1,fp8,fp8,0,0.0174346665541331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,1,128,1,float16,float16,0,0.017925333231687546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,1,128,1,float16,fp8,0,0.01814933369557063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,1,128,1,fp8,fp8,0,0.016976000120242436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,2,128,1,float16,float16,0,0.017871999492247898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,2,128,1,float16,fp8,0,0.018165333817402523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,4,128,1,float16,float16,0,0.016778666526079178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,4,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,1,128,1,float16,float16,0,0.017583999782800674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,4,128,1,float16,fp8,0,0.017103999853134155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,1,128,1,float16,fp8,0,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,256,4,2,128,1,fp8,fp8,0,0.016810666769742966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,1,128,1,fp8,fp8,0,0.01661866654952367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,2,128,1,float16,fp8,0,0.017743999759356182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,2,128,1,float16,float16,0,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,256,4,2,128,1,fp8,fp8,0,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,1,128,1,float16,fp8,0,0.10512000322341919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,1,128,1,fp8,fp8,0,0.09220266342163086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,1,128,1,float16,float16,0,0.10643733541170756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,2,128,1,float16,fp8,0,0.11286399761835735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,4,128,1,float16,fp8,0,0.07391466697057088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,4,128,1,float16,float16,0,0.07704533139864604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,2,128,1,float16,float16,0,0.15475733081499735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,128,4,2,128,1,fp8,fp8,0,0.09829866886138916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,1,128,1,float16,float16,0,0.05708266794681549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,4,128,1,fp8,fp8,0,0.06406400104363759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,1,128,1,fp8,fp8,0,0.0521066685517629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,1,128,1,float16,fp8,0,0.05700799822807312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,2,128,1,float16,float16,0,0.062181333700815834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,4,128,1,float16,float16,0,0.03975466638803482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,2,128,1,fp8,fp8,0,0.06620799998442332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,128,4,2,128,1,float16,fp8,0,0.06066666543483734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,4,128,1,float16,fp8,0,0.03809066613515218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,1,128,1,float16,float16,0,0.03332799921433131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,4,128,1,fp8,fp8,0,0.0371573343873024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,1,128,1,fp8,fp8,0,0.030720000465710957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,2,128,1,float16,float16,0,0.0344106654326121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,1,128,1,float16,fp8,0,0.03390933324893316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,4,128,1,float16,float16,0,0.02457600086927414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,4,128,1,float16,fp8,0,0.024442667762438457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,1,128,1,float16,float16,0,0.022997332115968067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,1,128,1,float16,fp8,0,0.02298133323589961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,4,128,1,fp8,fp8,0,0.02294933299223582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,2,128,1,fp8,fp8,0,0.033285332222779594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,128,4,2,128,1,float16,fp8,0,0.034703999757766724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,1,128,1,fp8,fp8,0,0.02176533391078313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,4,128,1,float16,float16,0,0.01923199991385142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,2,128,1,float16,float16,0,0.023621333142121632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,2,128,1,float16,fp8,0,0.023562667270501454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,128,4,2,128,1,fp8,fp8,0,0.021962667504946392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,1,128,1,float16,fp8,0,0.018629333625237148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,1,128,1,float16,float16,0,0.01817600056529045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,1,128,1,fp8,fp8,0,0.01833600054184596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,4,128,1,float16,fp8,0,0.018944000204404194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,4,128,1,fp8,fp8,0,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,2,128,1,float16,fp8,0,0.018496000518401463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,2,128,1,float16,float16,0,0.01868266612291336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,128,4,2,128,1,fp8,fp8,0,0.018709332992633183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,4,128,1,float16,float16,0,0.01747200017174085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,4,128,1,fp8,fp8,0,0.017157333592573803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,1,128,1,float16,float16,0,0.016890666137139004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,4,128,1,float16,fp8,0,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,1,128,1,fp8,fp8,0,0.017093333105246227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,2,128,1,float16,float16,0,0.017242666333913803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,2,128,1,float16,fp8,0,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,1,128,1,float16,fp8,0,0.017349333812793095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,4,128,1,float16,float16,0,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,128,4,2,128,1,fp8,fp8,0,0.017530667285124462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,4,128,1,float16,fp8,0,0.01651200031240781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,1,128,1,float16,fp8,0,0.016869333883126576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,1,128,1,fp8,fp8,0,0.01682666689157486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,2,128,1,float16,fp8,0,0.01695999999841054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,1,128,1,float16,float16,0,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,4,128,1,fp8,fp8,0,0.01651200031240781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,1,128,1,float16,float16,0,0.0162773331006368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,2,128,1,fp8,fp8,0,0.016858667135238647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,4,128,1,float16,fp8,0,0.016271999726692837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,128,4,2,128,1,float16,float16,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,4,128,1,fp8,fp8,0,0.01626666635274887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,4,128,1,float16,float16,0,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,1,128,1,float16,fp8,0,0.016704000532627106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,2,128,1,float16,float16,0,0.016154666741689045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,2,128,1,fp8,fp8,0,0.016421332955360413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,2,128,1,float16,fp8,0,0.016938666502634685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,128,4,1,128,1,fp8,fp8,0,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,4,128,1,fp8,fp8,0,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,4,128,1,float16,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,1,128,1,float16,fp8,0,0.0163680004576842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,4,128,1,float16,float16,0,0.015791999797026317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,1,128,1,fp8,fp8,0,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,2,128,1,float16,float16,0,0.016229332735141117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,1,128,1,float16,float16,0,0.015840000162522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,2,128,1,fp8,fp8,0,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,128,4,2,128,1,float16,fp8,0,0.01642666632930438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,1,128,1,float16,float16,0,0.06283733248710632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,1,128,1,float16,fp8,0,0.06203199923038483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,1,128,1,fp8,fp8,0,0.06665599842866261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,2,128,1,float16,float16,0,0.06888533135255177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,4,128,1,float16,float16,0,0.039994666973749794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,4,128,1,float16,fp8,0,0.0384853333234787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,2,128,1,float16,fp8,0,0.06749333441257477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,1,128,1,float16,float16,0,0.035829332967599235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,64,4,2,128,1,fp8,fp8,0,0.08018133540948232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,4,128,1,fp8,fp8,0,0.04478399952252706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,2,128,1,float16,float16,0,0.03762666632731756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,1,128,1,fp8,fp8,0,0.038160001238187156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,2,128,1,float16,fp8,0,0.03689600030581156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,4,128,1,float16,float16,0,0.024864000578721363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,4,128,1,fp8,fp8,0,0.026352000733216602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,1,128,1,float16,fp8,0,0.03606399893760681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,4,128,1,float16,fp8,0,0.02515200028816859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,64,4,2,128,1,fp8,fp8,0,0.040074666341145836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,1,128,1,float16,float16,0,0.024218666056791942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,1,128,1,float16,fp8,0,0.02399466683467229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,2,128,1,float16,float16,0,0.024319998919963837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,1,128,1,fp8,fp8,0,0.02500266581773758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,2,128,1,float16,fp8,0,0.02425066630045573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,4,128,1,fp8,fp8,0,0.01926400015751521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,4,128,1,float16,fp8,0,0.018789333601792652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,1,128,1,float16,fp8,0,0.017818666994571686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,4,128,1,float16,float16,0,0.018768000106016796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,1,128,1,float16,float16,0,0.017952000101407368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,64,4,2,128,1,fp8,fp8,0,0.025663999219735462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,2,128,1,float16,float16,0,0.018266666680574417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,1,128,1,fp8,fp8,0,0.018351999421914417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,4,128,1,float16,float16,0,0.015957333147525787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,2,128,1,float16,fp8,0,0.018218666315078735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,64,4,2,128,1,fp8,fp8,0,0.01865600049495697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,4,128,1,fp8,fp8,0,0.017317333569129307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,1,128,1,float16,fp8,0,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,1,128,1,fp8,fp8,0,0.016864000509182613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,4,128,1,float16,fp8,0,0.01969066634774208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,2,128,1,float16,float16,0,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,1,128,1,float16,float16,0,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,2,128,1,float16,fp8,0,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,64,4,2,128,1,fp8,fp8,0,0.01740266631046931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,4,128,1,float16,fp8,0,0.015589332828919092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,4,128,1,float16,float16,0,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,4,128,1,fp8,fp8,0,0.016229332735141117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,1,128,1,float16,fp8,0,0.01595199977358182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,1,128,1,float16,float16,0,0.01524266724785169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,2,128,1,float16,float16,0,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,2,128,1,float16,fp8,0,0.01591466615597407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,2,128,1,fp8,fp8,0,0.01659199967980385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,4,128,1,float16,fp8,0,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,4,128,1,fp8,fp8,0,0.01590399940808614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,4,128,1,float16,float16,0,0.01458666721979777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,64,4,1,128,1,fp8,fp8,0,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,1,128,1,float16,fp8,0,0.015279999623696009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,2,128,1,float16,float16,0,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,2,128,1,fp8,fp8,0,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,2,128,1,float16,fp8,0,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,1,128,1,float16,float16,0,0.014906667172908783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,4,128,1,float16,float16,0,0.014565333724021912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,64,4,1,128,1,fp8,fp8,0,0.016261332978804905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,1,128,1,float16,fp8,0,0.015247999380032221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,4,128,1,float16,fp8,0,0.014917333920796713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,4,128,1,fp8,fp8,0,0.015647999942302704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,1,128,1,float16,float16,0,0.01481066644191742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,1,128,1,fp8,fp8,0,0.01595199977358182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,2,128,1,float16,fp8,0,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,2,128,1,fp8,fp8,0,0.016095999628305435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,4,128,1,float16,float16,0,0.01403733342885971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,64,4,2,128,1,float16,float16,0,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,1,128,1,float16,float16,0,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,1,128,1,float16,fp8,0,0.015168000012636185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,4,128,1,fp8,fp8,0,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,1,128,1,fp8,fp8,0,0.0161013330022494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,2,128,1,float16,fp8,0,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,4,128,1,float16,fp8,0,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,2,128,1,float16,float16,0,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,64,4,2,128,1,fp8,fp8,0,0.016048000504573185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,1,128,1,float16,float16,0,0.04372266431649526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,1,128,1,float16,fp8,0,0.044437333941459656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,2,128,1,float16,float16,0,0.04612799982229868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,1,128,1,fp8,fp8,0,0.05186133086681366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,2,128,1,float16,fp8,0,0.04533333579699198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,4,128,1,float16,float16,0,0.027984000742435455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,32,4,2,128,1,fp8,fp8,0,0.05357866485913595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,4,128,1,fp8,fp8,0,0.033173332611719765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,1,128,1,float16,fp8,0,0.027765333652496338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,4,128,1,float16,fp8,0,0.02754666656255722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,1,128,1,float16,float16,0,0.02757333219051361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,1,128,1,fp8,fp8,0,0.03201066702604294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,2,128,1,float16,float16,0,0.028064000109831493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,2,128,1,float16,fp8,0,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,4,128,1,float16,float16,0,0.019952000429232914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,4,128,1,float16,fp8,0,0.020261333634455998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,1,128,1,fp8,fp8,0,0.022128000855445862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,1,128,1,float16,float16,0,0.01934933289885521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,1,128,1,float16,fp8,0,0.01960533360640208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,2,128,1,float16,float16,0,0.019626667102177937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,4,128,1,fp8,fp8,0,0.022511998812357586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,32,4,2,128,1,fp8,fp8,0,0.03271999955177307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,2,128,1,float16,fp8,0,0.019674666225910187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,4,128,1,float16,float16,0,0.016048000504573185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,32,4,2,128,1,fp8,fp8,0,0.02258666604757309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,4,128,1,float16,fp8,0,0.015957333147525787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,1,128,1,float16,float16,0,0.01534933348496755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,2,128,1,float16,fp8,0,0.015743999431530636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,4,128,1,fp8,fp8,0,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,1,128,1,fp8,fp8,0,0.017344000438849132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,2,128,1,float16,float16,0,0.015429332852363586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,2,128,1,fp8,fp8,0,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,32,4,1,128,1,float16,fp8,0,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,4,128,1,float16,fp8,0,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,4,128,1,float16,float16,0,0.014554666976133982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,4,128,1,fp8,fp8,0,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,1,128,1,float16,fp8,0,0.01524266724785169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,1,128,1,fp8,fp8,0,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,1,128,1,float16,float16,0,0.014783999572197596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,2,128,1,float16,float16,0,0.014682666709025701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,2,128,1,fp8,fp8,0,0.016719999412695568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,32,4,2,128,1,float16,fp8,0,0.015103999525308609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,4,128,1,float16,fp8,0,0.014560000350077948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,4,128,1,float16,float16,0,0.014122666170199713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,4,128,1,fp8,fp8,0,0.016165333489576977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,1,128,1,float16,float16,0,0.014303999642531076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,1,128,1,float16,fp8,0,0.01469333345691363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,1,128,1,fp8,fp8,0,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,2,128,1,float16,float16,0,0.0144213338692983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,2,128,1,fp8,fp8,0,0.0161920003592968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,4,128,1,float16,float16,0,0.013754667093356451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,4,128,1,fp8,fp8,0,0.01597333326935768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,4,128,1,float16,fp8,0,0.014218666901191076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,32,4,2,128,1,float16,fp8,0,0.017653333644072216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,2,128,1,float16,float16,0,0.01404800017674764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,1,128,1,float16,fp8,0,0.014688000082969666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,1,128,1,fp8,fp8,0,0.016197333733240765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,1,128,1,float16,float16,0,0.014080000420411428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,2,128,1,float16,fp8,0,0.014618666221698126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,32,4,2,128,1,fp8,fp8,0,0.01603200038274129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,4,128,1,float16,float16,0,0.013440000514189402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,4,128,1,fp8,fp8,0,0.015722667177518208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,4,128,1,float16,fp8,0,0.013898666948080063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,1,128,1,float16,fp8,0,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,1,128,1,float16,float16,0,0.013914667069911957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,2,128,1,float16,float16,0,0.013962666193644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,1,128,1,fp8,fp8,0,0.015717333803574245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,2,128,1,float16,fp8,0,0.014416000495354334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,32,4,2,128,1,fp8,fp8,0,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,4,128,1,float16,float16,0,0.01328533391157786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,4,128,1,fp8,fp8,0,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,1,128,1,float16,float16,0,0.013904000322024027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,4,128,1,float16,fp8,0,0.013957332819700241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,1,128,1,float16,fp8,0,0.014463999619086584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,2,128,1,fp8,fp8,0,0.01597333326935768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,2,128,1,float16,float16,0,0.013967999567588171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,1,128,1,fp8,fp8,0,0.015781333049138386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,32,4,2,128,1,float16,fp8,0,0.014597332725922266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,1,128,1,float16,fp8,0,0.03974399964014689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,2,128,1,fp8,fp8,0,0.04684266448020935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,1,128,1,fp8,fp8,0,0.046522667010625206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,2,128,1,float16,fp8,0,0.039887999494870506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,4,128,1,float16,fp8,0,0.024677333732446034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,1,128,1,float16,float16,0,0.03975466638803482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,4,128,1,float16,float16,0,0.024256000916163128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,4,128,1,fp8,fp8,0,0.03002133220434189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,256,16,4,2,128,1,float16,float16,0,0.040048000713189445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,1,128,1,float16,float16,0,0.02492800106604894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,1,128,1,fp8,fp8,0,0.029461334149042766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,2,128,1,float16,float16,0,0.025258667767047882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,1,128,1,float16,fp8,0,0.025610665480295818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,4,128,1,float16,float16,0,0.01834133391578992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,2,128,1,fp8,fp8,0,0.02939733366171519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,128,16,4,2,128,1,float16,fp8,0,0.025514667232831318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,4,128,1,fp8,fp8,0,0.02086399992307027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,1,128,1,float16,fp8,0,0.018197332819302876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,1,128,1,float16,float16,0,0.017792000124851864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,4,128,1,float16,fp8,0,0.018239999810854595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,1,128,1,fp8,fp8,0,0.02067733307679494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,2,128,1,float16,fp8,0,0.018538666268189747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,2,128,1,float16,float16,0,0.01791999985774358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,4,128,1,float16,float16,0,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,64,16,4,2,128,1,fp8,fp8,0,0.020970667401949566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,4,128,1,float16,fp8,0,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,4,128,1,fp8,fp8,0,0.01684800038735072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,2,128,1,float16,fp8,0,0.015178666760524115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,1,128,1,float16,fp8,0,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,1,128,1,fp8,fp8,0,0.016986666868130367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,2,128,1,float16,float16,0,0.01488000030318896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,1,128,1,float16,float16,0,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,4,128,1,float16,float16,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,32,16,4,2,128,1,fp8,fp8,0,0.01700266698996226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,4,128,1,fp8,fp8,0,0.016442666451136272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,4,128,1,float16,fp8,0,0.014645333091417948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,1,128,1,fp8,fp8,0,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,1,128,1,float16,fp8,0,0.014783999572197596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,1,128,1,float16,float16,0,0.014202666779359182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,2,128,1,float16,float16,0,0.014303999642531076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,2,128,1,float16,fp8,0,0.014757333944241205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,16,16,4,2,128,1,fp8,fp8,0,0.016517333686351776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,4,128,1,float16,float16,0,0.013946666071812311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,4,128,1,float16,fp8,0,0.014373333503802618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,4,128,1,fp8,fp8,0,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,2,128,1,float16,float16,0,0.014250667144854864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,1,128,1,float16,float16,0,0.014032000054915747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,2,128,1,float16,fp8,0,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,1,128,1,fp8,fp8,0,0.01616000011563301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,2,128,1,fp8,fp8,0,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,4,128,1,float16,float16,0,0.013754667093356451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,8,16,4,1,128,1,float16,fp8,0,0.014720000326633453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,4,128,1,float16,fp8,0,0.014389333625634512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,4,128,1,fp8,fp8,0,0.016010666886965435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,2,128,1,float16,fp8,0,0.0145066666106383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,2,128,1,float16,float16,0,0.014085333794355392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,1,128,1,float16,float16,0,0.014090667168299357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,1,128,1,fp8,fp8,0,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,1,128,1,float16,fp8,0,0.014805333067973455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,4,128,1,float16,float16,0,0.013418667018413544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,4,16,4,2,128,1,fp8,fp8,0,0.016149333367745083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,4,128,1,float16,fp8,0,0.01394133393963178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,4,128,1,fp8,fp8,0,0.01575999955336253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,1,128,1,float16,float16,0,0.014042666802803675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,1,128,1,float16,fp8,0,0.014261333892742792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,1,128,1,fp8,fp8,0,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,2,128,1,float16,fp8,0,0.01422400027513504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,2,128,1,float16,float16,0,0.014271999398867289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,4,128,1,float16,float16,0,0.013461332768201828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,2,16,4,2,128,1,fp8,fp8,0,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,4,128,1,float16,fp8,0,0.013797332843144735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,4,128,1,fp8,fp8,0,0.01569066693385442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,1,128,1,fp8,fp8,0,0.015882667154073715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,1,128,1,float16,fp8,0,0.014357333381970724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,2,128,1,float16,float16,0,0.014101333916187286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,1,128,1,float16,float16,0,0.01358933374285698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,2,128,1,float16,fp8,0,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,context_attention,1,16,4,2,128,1,fp8,fp8,0,0.01598400001724561
