framework,version,device,op_name,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,1,0.025861332813898723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,15,0.03489600121974945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,31,0.03506666670242945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,63,0.03620799879233042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,63,0.03538133452335993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,127,0.04776533444722494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,127,0.043023998538653054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,3,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,7,0.02790933350721995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,3,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,1,0.026416001220544178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,15,0.03497066597143809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,31,0.035173334181308746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,7,0.02824000020821889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,511,0.13806933164596558
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,511,0.1146613359451294
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,1023,0.2611200014750163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,255,0.07197866837183635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,255,0.06922666728496552
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,1023,0.2180266578992208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,2047,0.5031520128250122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,float16,4095,0.9879946708679199
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,1,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,4095,0.8111999829610189
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,1,0.009839999799927076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,64,128,1,float16,fp8,2047,0.41681599617004395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,3,0.009301333377758661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,3,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,7,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,7,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,15,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,31,0.010415999839703241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,15,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,31,0.010357333347201347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,63,0.012149333953857422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,63,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,127,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,255,0.012272000312805176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,255,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,127,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,511,0.016741332908471424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,511,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,1023,0.029690665503342945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,2047,0.03766933331886927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,1023,0.029450667401154835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,2047,0.037258667250474296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,4095,0.05622933308283488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,4095,0.052330667773882546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,8191,0.09777067104975383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,8191,0.08716799815495808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,float16,16383,0.16883200407028198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,64,128,1,float16,fp8,16383,0.1518400013446808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,1,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,3,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,1,0.009925333162148794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,3,0.010048000141978264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,7,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,7,0.009888000165422758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,15,0.010191999996701876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,15,0.009989333028594652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,31,0.010608000059922537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,127,0.012341332932313284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,63,0.012223999947309494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,31,0.010586666564146677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,255,0.012597333639860153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,127,0.012330666184425354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,255,0.012426666915416718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,511,0.028325334191322327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,511,0.028213332096735638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,1023,0.033029332756996155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,1023,0.03278400003910065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,2047,0.048613334695498146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,2047,0.04427200059096018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,8191,0.14602133631706238
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,4095,0.07196266452471416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,4095,0.08576533198356628
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,8191,0.1241973340511322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,1,0.04788800080617269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,float16,16383,0.2675146659215291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,64,128,1,float16,fp8,16383,0.22138132651646933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,1,0.04915200173854828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,3,0.05109866460164388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,7,0.05276266733805338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,15,0.06452799836794536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,3,0.049973333875338234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,7,0.0517493337392807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,15,0.06453333298365276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,31,0.06679466863473256
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,31,0.06461333235104878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,63,0.0721013347307841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,63,0.06674133241176605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,255,0.1335093379020691
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,127,0.08666666348775227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,127,0.08783466617266338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,255,0.13091199596722922
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,511,0.26097599665323895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,511,0.21916800737380981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,1023,0.42022399107615155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,1023,0.5088373422622681
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,float16,2047,0.9908746878306071
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,64,128,1,float16,fp8,2047,0.8041919867197672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,1,0.00891733355820179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,1,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,3,0.009642666826645533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,3,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,7,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,7,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,31,0.011781333635250727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,15,0.009861333295702934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,63,0.01173866664369901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,15,0.010048000141978264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,31,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,63,0.012517333030700684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,127,0.012069333344697952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,127,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,255,0.01392000044385592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,255,0.013631999492645264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,511,0.019541333119074505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,1023,0.03027733415365219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,2047,0.0714026689529419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,511,0.01915733392039935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,1023,0.03984533250331879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,2047,0.06062933305899302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,4095,0.13269333044687906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,4095,0.11013333002726237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,8191,0.2555306752522786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,8191,0.20833067099253336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,float16,16383,0.5064906676610311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,64,128,1,float16,fp8,16383,0.4059520165125529
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,1,0.09101333220799764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,1,0.09366400043169658
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,3,0.09434133768081665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,7,0.09805867075920105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,15,0.1267306705315908
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,3,0.09675733248392741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,7,0.10039466619491577
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,15,0.12378666798273723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,31,0.13362666964530945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,31,0.1292586624622345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,63,0.13592533270517984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,63,0.1356000006198883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,127,0.1668000022570292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,127,0.16555200020472208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,255,0.2565973401069641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,511,0.4989866813023885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,511,0.423306663831075
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,255,0.2523519992828369
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,float16,1023,0.9929440021514893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,64,128,1,float16,fp8,1023,0.8149279753367106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,1,0.17705599466959634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,3,0.18547199169794717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,3,0.1880213419596354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,1,0.18182400862375894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,7,0.20532800753911337
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,7,0.1987946629524231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,15,0.25251734256744385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,15,0.2537546753883362
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,31,0.2590559919675191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,63,0.26335465908050537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,31,0.25996800263722736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,63,0.263482669989268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,127,0.32500267028808594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,127,0.32371199131011963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,float16,255,0.5003840128580729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,64,128,1,float16,fp8,255,0.495471994082133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,1,0.3739733298619588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,1,0.3729493220647176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,3,0.39153067270914715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,3,0.3960533142089844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,7,0.40812798341115314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,7,0.418229341506958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,15,0.4979519844055176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,15,0.5055679877599081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,31,0.5112586816151937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,31,0.5130346616109213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,63,0.5184319814046224
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,63,0.5195146799087524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,1,0.014943999548753103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,float16,127,0.6404693524042765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,1,0.015397333850463232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,64,128,1,float16,fp8,127,0.639514684677124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,3,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,3,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,7,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,7,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,15,0.01655999943614006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,63,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,15,0.01700266698996226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,31,0.019813333948453266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,31,0.02012266715367635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,63,0.020282667130231857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,255,0.02386133372783661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,127,0.02025066688656807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,127,0.02027733375628789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,255,0.024442667762438457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,511,0.040864000717798867
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,511,0.035360001027584076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,1023,0.07457066575686137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,2047,0.13667200009028116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,1023,0.062128002444903054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,2047,0.11468799908955891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,4095,0.25712533791859943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,4095,0.2135093410809835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,8191,0.5005600055058798
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,8191,0.4081333478291829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,fp8,16383,0.7990720272064209
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,64,128,1,float16,float16,16383,1.000501314798991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,1,0.7520853678385416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,1,0.7734346389770508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,3,0.7765920162200928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,3,0.7946453094482422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,7,0.8092373212178549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,7,0.8307519753774008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,15,0.9890453020731608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,15,1.0041066805521648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,31,1.01529065767924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,31,1.0191360314687092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,float16,63,1.0291413466135662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,64,128,1,float16,fp8,63,1.0319519837697346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,float16,1,1.4962399800618489
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,fp8,1,1.5400746663411458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,fp8,3,1.5819946924845378
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,float16,3,1.5458879470825195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,float16,7,1.6122080485026042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,fp8,7,1.6547147432963054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,float16,15,1.9719680150349934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,fp8,15,2.002138614654541
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,1,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,3,0.015487999965747198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,3,0.015482666591803232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,7,0.01594666639963786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,7,0.016234666109085083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,float16,31,2.0230026245117188
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,15,0.019637333850065868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,15,0.019472000499566395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,31,0.01951466624935468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,63,0.01974933346112569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,63,0.019797333826621372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,127,0.024154665569464367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,64,128,1,float16,fp8,31,2.0320639610290527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,31,0.0195573332409064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,127,0.02351466566324234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,255,0.03513599932193756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,511,0.07468800246715546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,511,0.06092800199985504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,255,0.03925866633653641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,1023,0.13609600067138672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,1023,0.11549333731333415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,2047,0.25726934274037677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,2047,0.21842666467030844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,4095,0.5013333161671957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,4095,0.4195733467737834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,float16,8191,1.0450026988983154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,1,0.020661332954963047
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,64,128,1,float16,fp8,8191,0.8335733413696289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,1,0.02088533341884613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,3,0.021594665944576263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,7,0.021962667504946392
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,7,0.0223786657055219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,15,0.02739199995994568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,15,0.027701333165168762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,31,0.027552001178264618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,3,0.02111999938885371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,63,0.028218666712443035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,63,0.02762666592995326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,127,0.03682133307059606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,127,0.03296533226966858
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,31,0.027632000545660656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,255,0.056176001826922096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,255,0.053599998354911804
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,511,0.08825066685676575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,1023,0.19901333252588907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,511,0.10699199636777242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,2047,0.3800533215204875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,1023,0.16729066769282022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,float16,4095,0.7453599770863851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,2047,0.31866133213043213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,3,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,1,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,3,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,48,48,128,1,float16,fp8,4095,0.6181226571400961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,1,0.009525333220760027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,7,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,15,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,15,0.009583999713261923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,31,0.010304000228643417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,127,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,63,0.012106666962305704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,255,0.012351999680201212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,63,0.012037333101034164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,127,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,255,0.0122079998254776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,511,0.01659199967980385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,511,0.016602666427691776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,1023,0.029520000020662945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,1023,0.02943466603755951
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,2047,0.03730133424202601
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,4095,0.0514933317899704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,4095,0.05412266651789347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,8191,0.09515200058619182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,2047,0.037151999771595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,8191,0.08397333820660909
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,float16,16383,0.16621866822242737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,1,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,48,48,128,1,float16,fp8,16383,0.1504533290863037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,1,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,3,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,3,0.009797333429257074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,15,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,7,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,15,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,7,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,63,0.012341332932313284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,31,0.010640000303586325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,63,0.012122667084137598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,127,0.012229333321253458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,127,0.012191999703645706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,255,0.012639999389648438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,511,0.02779199928045273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,511,0.027829334139823914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,1023,0.032373333970705666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,1023,0.03241066634654999
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,2047,0.04593066871166229
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,255,0.01227733368674914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,2047,0.043706665436426796
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,4095,0.07764799892902374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,4095,0.0693333347638448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,8191,0.13223466277122498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,8191,0.1200213332970937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,fp8,16383,0.21692800521850586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,48,48,128,1,float16,float16,16383,0.23844265937805176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,1,0.03812800099452337
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,1,0.037290667494138084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,3,0.03865066667397817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,3,0.03947199881076813
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,15,0.050106664498647056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,7,0.040234667559464775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,7,0.04081599911053976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,15,0.050517335534095764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,127,0.06797866523265839
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,63,0.050714666644732155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,63,0.055999999245007835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,31,0.050666665037473045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,255,0.10326932867368062
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,127,0.06679999828338623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,31,0.05038933455944061
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,255,0.10095466176668803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,511,0.20007999738057455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,511,0.1675306757291158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,1023,0.38650667667388916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,1023,0.3199946681658427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,3,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,float16,2047,0.747920036315918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,48,48,128,1,float16,fp8,2047,0.6135520140329996
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,1,0.009258666386206945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,1,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,3,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,15,0.009712000067035357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,7,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,7,0.009503999724984169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,15,0.010064000263810158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,31,0.011493333925803503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,31,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,127,0.01180800050497055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,63,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,127,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,63,0.011701333026091257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,255,0.013669333110253016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,255,0.013605333864688873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,511,0.019189332922299702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,1023,0.031370667119820915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,511,0.01933866615096728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,1023,0.030192000170548756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,2047,0.06268266836802165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,8191,0.20542399088541666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,4095,0.11385066310564677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,4095,0.10920000076293945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,2047,0.0529120018084844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,8191,0.21890133619308472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,float16,16383,0.4264800151189168
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,48,48,128,1,float16,fp8,16383,0.4012853304545085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,1,0.07142400244871776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,1,0.06949866811434428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,3,0.0722453345855077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,3,0.07411733269691467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,15,0.0944160024325053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,7,0.07660800218582153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,15,0.09452266494433086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,7,0.07448533177375793
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,31,0.10152533650398254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,31,0.0953493316968282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,63,0.10391466816266377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,63,0.10335999727249146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,127,0.12737600008646646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,127,0.1262986660003662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,511,0.3776373465855916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,255,0.19482133785883585
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,511,0.32078399260838825
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,255,0.19155732790629068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,float16,1023,0.7488106886545817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,3,0.13945600390434265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,1,0.13447466492652893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,48,48,128,1,float16,fp8,1023,0.6160853306452433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,1,0.13844799995422363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,3,0.14256532986958823
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,7,0.14958932995796204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,15,0.18758400281270346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,7,0.1490079959233602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,15,0.19149333238601685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,31,0.19669866561889648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,63,0.19952533642450967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,31,0.19693867365519205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,63,0.19990400473276773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,127,0.24634132782618204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,127,0.24457067251205444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,fp8,255,0.3741866747538249
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,1,0.27142399549484253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,1,0.26794666051864624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,3,0.2922933300336202
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,3,0.2866453329722087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,48,48,128,1,float16,float16,255,0.3782613277435303
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,7,0.30909866094589233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,7,0.31270400683085126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,15,0.37588798999786377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,15,0.3816106716791789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,31,0.3859359820683797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,31,0.38712533315022785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,63,0.3913280169169108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,float16,127,0.48321600755055744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,63,0.39164801438649494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,1,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,1,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,48,48,128,1,float16,fp8,127,0.4816853205362956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,3,0.015130666395028433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,3,0.015311999867359797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,7,0.014853333433469137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,15,0.015253332753976187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,7,0.01534933348496755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,15,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,31,0.01684800038735072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,31,0.01699200024207433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,127,0.017242666333913803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,63,0.017125333348910015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,63,0.01718933383623759
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,127,0.01758933315674464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,255,0.020224000016848247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,1023,0.06337066491444905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,511,0.02844800055027008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,511,0.03253866732120514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,255,0.020117333779732387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,1023,0.05133333305517832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,2047,0.11357866724332173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,4095,0.18158932526906332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,4095,0.21268266439437866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,2047,0.09265599648157756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,8191,0.4116213321685791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,8191,0.34599467118581134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,fp8,16383,0.6813066800435384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,48,48,128,1,float16,float16,16383,0.8640906810760498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,1,0.5832639932632446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,1,0.5669653415679932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,3,0.5854293505350748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,3,0.598906675974528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,7,0.6099146604537964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,15,0.7454026540120443
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,7,0.6262933413187662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,15,0.7558186848958334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,31,0.7649172941843668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,31,0.7673493226369222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,float16,63,0.7743573188781738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,48,48,128,1,float16,fp8,63,0.7763466835021973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,float16,1,1.1263999938964844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,fp8,1,1.1591733296712239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,float16,3,1.1637760003407795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,fp8,3,1.189680020014445
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,float16,7,1.213205337524414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,fp8,7,1.2458613713582356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,float16,15,1.483791987101237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,fp8,15,1.5058453877766926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,1,0.014181333283583323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,3,0.01452800010641416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,1,0.014293332894643148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,3,0.014511999984582266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,7,0.014495999862750372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,float16,31,1.5219093958536785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,7,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,15,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,15,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,48,48,128,1,float16,fp8,31,1.5284105936686199
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,31,0.01628799984852473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,255,0.031162666777769726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,63,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,31,0.016127999871969223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,63,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,127,0.01905599981546402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,127,0.01899733394384384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,255,0.027456000447273254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,511,0.04915733138720194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,511,0.0626933326323827
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,1023,0.11641599734624226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,2047,0.21808000405629477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,2047,0.1814346710840861
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,4095,0.42324264844258624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,1023,0.09175999959309895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,float16,8191,0.8308800061543783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,4095,0.34728535016377765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,1,0.019904000063737232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,1,0.0194560003777345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,3,0.01979200045267741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,48,48,128,1,float16,fp8,8191,0.6931573549906412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,7,0.020224000016848247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,3,0.02033599962790807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,7,0.0199946661790212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,15,0.02365333338578542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,15,0.02348800003528595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,63,0.023893333971500397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,31,0.0236160010099411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,31,0.023823998868465424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,63,0.023770667612552643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,127,0.030495998760064442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,127,0.0281333327293396
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,511,0.09471999605496724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,255,0.04941866795221964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,255,0.04412800073623657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,511,0.07668266693751018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,1023,0.17801600694656372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,1023,0.14193066954612732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,4095,0.6504640181859335
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,float16,2047,0.33444265524546307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,1,0.009301333377758661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,4095,0.531770666440328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,40,40,128,1,float16,fp8,2047,0.27645333607991535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,3,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,1,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,3,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,7,0.009488000224033991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,15,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,7,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,15,0.01003200002014637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,31,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,31,0.010351999973257383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,63,0.012058666596810022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,127,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,63,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,255,0.012250666817029318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,255,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,511,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,511,0.016688000410795212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,2047,0.0322773332397143
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,1023,0.017866666118303936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,2047,0.03230933348337809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,1023,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,4095,0.04388799766699473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,8191,0.07647466659545898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,4095,0.04368533194065094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,float16,16383,0.12998933593432108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,8191,0.06719466547171275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,1,0.00997866690158844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,1,0.009632000078757605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,3,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,40,40,128,1,float16,fp8,16383,0.11928533514340718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,3,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,7,0.009632000078757605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,7,0.009930666536092758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,15,0.009706666693091393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,31,0.010453333457310995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,15,0.010058666889866194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,31,0.010384000216921171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,63,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,255,0.012213333199421564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,63,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,127,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,255,0.012527999778588613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,511,0.02777066578467687
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,1023,0.032298666735490165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,511,0.027642667293548584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,1023,0.03201066702604294
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,2047,0.04363733530044556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,2047,0.04409599800904592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,4095,0.07657066484292348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,4095,0.0661653329928716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,8191,0.11931733290354411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,8191,0.12973866860071817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,float16,16383,0.23662932713826498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,7,0.0342399999499321
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,3,0.033626665671666466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,1,0.03244800120592117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,3,0.033002667129039764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,1,0.031845333675543465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,7,0.03467733412981033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,40,40,128,1,float16,fp8,16383,0.2134666641553243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,15,0.042810668547948204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,31,0.04288533329963684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,31,0.04292800029118856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,15,0.0425600012143453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,63,0.046282668908437095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,127,0.05791999896367391
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,255,0.08785600463549297
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,63,0.04330133398373922
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,255,0.08517332871754964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,511,0.1681493322054545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,127,0.05457599957784017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,511,0.14149866501490274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,1023,0.32443199555079144
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,1023,0.2702239950497945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,float16,2047,0.6264533201853434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,1,0.01586666703224182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,40,40,128,1,float16,fp8,2047,0.515343983968099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,1,0.016506666938463848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,7,0.015850666910409927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,3,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,15,0.016234666109085083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,7,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,15,0.016800000021855038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,31,0.01785600061217944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,3,0.015717333803574245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,31,0.017909333109855652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,127,0.021274665991465252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,63,0.02139200021823247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,127,0.021136000752449036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,255,0.02160000056028366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,63,0.021221332252025604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,255,0.0215786670645078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,1023,0.03688533355792364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,511,0.024703999360402424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,2047,0.05654400090376536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,1023,0.03498666733503342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,511,0.024666666984558105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,2047,0.0710506687561671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,4095,0.124208003282547
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,4095,0.10804800192515056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,8191,0.23226133982340494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,float16,16383,0.44554134209950763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,16383,0.3779946565628052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,1,0.06028800209363302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,1,0.05868266522884369
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,3,0.06100266675154368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,40,40,128,1,float16,fp8,8191,0.19694934288660684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,7,0.0626986672480901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,3,0.06262933214505513
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,7,0.06423999865849812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,15,0.07922133306662242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,15,0.07963733375072479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,31,0.08460799853006999
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,31,0.07967466612656911
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,63,0.08776533603668213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,63,0.08572799960772197
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,127,0.10618666807810466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,127,0.10730133454004924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,255,0.16407466928164163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,255,0.1606613298257192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,1023,0.5153973499933878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,511,0.3169386585553487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,fp8,511,0.2672853271166484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,40,40,128,1,float16,float16,1023,0.6256639957427979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,1,0.1127839982509613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,1,0.1165066659450531
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,3,0.11723732948303223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,7,0.12283200025558472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,3,0.11990933616956075
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,7,0.12486400206883748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,31,0.16539733608563742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,15,0.16010133425394693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,15,0.15528000394503275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,31,0.1644373337427775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,63,0.1679146687189738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,63,0.16751466194788614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,255,0.3177013397216797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,float16,127,0.20661866664886475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,127,0.2053013245264689
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,40,40,128,1,float16,fp8,255,0.3129813273747762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,1,0.22114666302998862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,1,0.22684800624847412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,7,0.2582933306694031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,3,0.2348853349685669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,3,0.23761600255966187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,7,0.2558719913164775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,15,0.31491732597351074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,15,0.3190293312072754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,31,0.3230133255322774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,31,0.3240106701850891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,63,0.32756266991297406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,63,0.3279520074526469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,float16,127,0.40383466084798175
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,1,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,1,0.014981333166360855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,3,0.014181333283583323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,40,40,128,1,float16,fp8,127,0.4027786652247111
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,7,0.01414399966597557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,3,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,7,0.014501333236694336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,15,0.01492799942692121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,15,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,31,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,31,0.016885332763195038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,63,0.016997333616018295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,127,0.01714133347074191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,63,0.01692266638080279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,127,0.01720533271630605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,255,0.020074666788180668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,1023,0.04731200138727824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,1023,0.05973866581916809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,255,0.020117333779732387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,511,0.028079998989899952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,511,0.03018666555484136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,2047,0.10550399621327718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,2047,0.09187199672063191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,4095,0.16940265893936157
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,8191,0.3901173273722331
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,4095,0.20137600104014078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,8191,0.3402880032857259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,float16,16383,0.7721866766611735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,40,40,128,1,float16,fp8,16383,0.6633919874827067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,1,0.47409598032633465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,1,0.485642671585083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,3,0.5002133448918661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,3,0.4896746476491292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,7,0.5102293491363525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,15,0.6226773262023926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,7,0.5240319967269897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,15,0.632698655128479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,31,0.6395839850107828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,31,0.6418559948603312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,float16,63,0.6463253498077393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,40,40,128,1,float16,fp8,63,0.6486506859461466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,float16,1,0.9410346349080404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,fp8,1,0.9674452940622965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,float16,3,0.9715147018432617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,fp8,3,0.9941279888153076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,float16,7,1.014682690302531
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,fp8,7,1.0414613087972004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,float16,15,1.238810698191325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,fp8,15,1.2591040134429932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,1,0.013834666460752487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,1,0.013909333695967993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,3,0.013967999567588171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,7,0.014010666559139887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,7,0.014106666048367819
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,3,0.014453332871198654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,float16,31,1.271386702855428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,15,0.015263999501864115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,40,40,128,1,float16,fp8,31,1.2761759757995605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,15,0.01533866673707962
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,31,0.015189333508412043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,31,0.015376000354687372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,63,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,63,0.015589332828919092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,127,0.017690667261679966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,127,0.01800000046690305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,255,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,511,0.0447573314110438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,255,0.02475733309984207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,511,0.05850133299827576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,1023,0.10641066233317058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,1023,0.08973866701126099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,2047,0.16514133413632712
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,4095,0.32917332649230957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,2047,0.20061333974202475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,1,0.014917333920796713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,4095,0.38843735059102374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,fp8,8191,0.6542773246765137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,40,40,128,1,float16,float16,8191,0.7841653029123942
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,1,0.015295999745527903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,3,0.01552533358335495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,7,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,3,0.015546667079130808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,7,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,15,0.019530666371186573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,15,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,31,0.019578666736682255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,63,0.019695999721686046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,63,0.019845332950353622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,31,0.019626667102177937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,127,0.023423999547958374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,255,0.03919466584920883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,127,0.024304000039895374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,255,0.03569599986076355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,511,0.061050668358802795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,1023,0.13643200198809305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,511,0.0741066684325536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,1023,0.11565333604812622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,2047,0.21784534056981406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,2047,0.25755200783411664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,4095,0.5017386674880981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,float16,8191,1.0502453645070393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,8191,0.836522658665975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,32,128,1,float16,fp8,4095,0.4198506673177083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,1,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,1,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,3,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,15,0.009663999701539675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,3,0.00979200005531311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,7,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,15,0.009824000298976898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,31,0.01028266673286756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,31,0.010368000095089277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,63,0.011962667107582092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,127,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,63,0.011968000481526056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,127,0.011952000359694162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,255,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,1023,0.017418666432301205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,255,0.012069333344697952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,511,0.01651200031240781
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,1023,0.017610666652520496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,2047,0.029605334003766377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,511,0.016645333419243496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,2047,0.029653333127498627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,4095,0.040607998768488564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,4095,0.04008000095685323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,16383,0.10937600334485371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,8191,0.06819200019041698
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,8191,0.05946666498978933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,16383,0.10043733318646748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,fp8,32767,0.17443732420603433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,32,128,1,float16,float16,32767,0.19443732500076294
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,1,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,3,0.009519999846816063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,1,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,3,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,7,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,7,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,15,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,31,0.010314666976531347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,31,0.010442666709423065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,15,0.010101333260536194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,63,0.012335999558369318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,63,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,511,0.016869333883126576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,127,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,255,0.01231466606259346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,255,0.01240533341964086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,511,0.016805333395799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,2047,0.03766933331886927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,1023,0.029658667743206024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,1023,0.029696000119050343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,2047,0.0373333344856898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,4095,0.0565226674079895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,4095,0.05221333106358846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,8191,0.08745599786440532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,8191,0.09738133351008098
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,16383,0.16942399740219116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,16383,0.1523360013961792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,1,0.02606400102376938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,fp8,32767,0.28045332431793213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,32,128,1,float16,float16,32767,0.3129013379414876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,1,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,3,0.0271573339899381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,3,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,7,0.028197333216667175
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,15,0.03498133271932602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,7,0.02863999952872594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,15,0.03517866631348928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,31,0.03558400024970373
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,31,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,63,0.03677333394686381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,127,0.04784533381462097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,63,0.03547733277082443
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,127,0.043162668744723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,255,0.07213866710662842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,255,0.06962666908899943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,511,0.13844266533851624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,511,0.11515200138092041
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,1023,0.21862399578094482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,2047,0.4180213212966919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,1023,0.2617759903271993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,2047,0.5031840006510416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,float16,4095,0.9886613686879476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,3,0.0100426667680343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,3,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,32,128,1,float16,fp8,4095,0.8116693496704102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,1,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,1,0.009775999933481216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,7,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,7,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,15,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,31,0.010469333579142889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,63,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,63,0.01251199965675672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,127,0.012565333396196365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,127,0.012346666306257248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,255,0.012762666990359625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,511,0.028192001084486645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,255,0.01239466667175293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,511,0.02844800055027008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,1023,0.03286399940649668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,2047,0.048901334404945374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,4095,0.08641599615414937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,2047,0.04419200122356415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,1023,0.03294933338960012
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,4095,0.07206933200359344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,8191,0.14724266529083252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,8191,0.12478933731714885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,1,0.049269333481788635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,16383,0.2219840089480082
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,16383,0.268832008043925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,1,0.04809066653251648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,float16,32767,0.5107359886169434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,32,128,1,float16,fp8,32767,0.4177653392155965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,3,0.05013333261013031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,7,0.051685333251953125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,3,0.05147199829419454
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,31,0.0674773355325063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,7,0.0529013325770696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,15,0.06479999919732411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,31,0.06503466765085857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,15,0.06491200129191081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,63,0.07233066856861115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,63,0.067071999112765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,127,0.0867039958635966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,127,0.08816533287366231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,255,0.13384532928466797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,255,0.1313813328742981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,511,0.25949867566426593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,511,0.21968533595403036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,1023,0.5087253252665201
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,1023,0.41989866892496747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,float16,2047,0.991370677947998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,32,128,1,float16,fp8,2047,0.8064746856689453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,1,0.09168533484141032
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,1,0.0940053363641103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,3,0.09483733773231506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,7,0.10088533163070679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,7,0.09868266185124715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,3,0.09703999757766724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,15,0.126991997162501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,15,0.1251520017782847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,31,0.13382933537165323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,31,0.12924800316492716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,63,0.13563733299573263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,63,0.13594667116800943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,255,0.2566186587015788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,127,0.16725865999857584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,127,0.16616533199946085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,255,0.2534240086873372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,1,0.17747199535369873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,float16,511,0.49512000878651935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,3,0.1859253247578939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,1,0.1825066606203715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,32,128,1,float16,fp8,511,0.42350931962331134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,3,0.18803733587265015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,7,0.20588266849517822
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,15,0.2534826596577962
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,31,0.2606293360392253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,7,0.19925334056218466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,15,0.2547626694043477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,63,0.26368000109990436
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,63,0.2637919982274373
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,31,0.26102399826049805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,127,0.3254879911740621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,127,0.3241013288497925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,float16,255,0.5005813439687093
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,1,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,32,128,1,float16,fp8,255,0.49615466594696045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,1,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,3,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,7,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,3,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,15,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,7,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,15,0.010319999729593595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,31,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,31,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,63,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,63,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,127,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,127,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,255,0.013690666606028875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,255,0.013829333086808523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,511,0.01937599976857503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,1023,0.04046933352947235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,1023,0.030378667016824085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,511,0.019498666127522785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,2047,0.07155199845631917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,4095,0.1328000028928121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,2047,0.06061866879463196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,4095,0.11035733421643575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,8191,0.2552746733029683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,8191,0.20874667167663574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,16383,0.5060373147328695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,float16,32767,1.2652053038279216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,16383,0.4071679910024007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,32,128,1,float16,fp8,32767,1.2292959690093994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,1,0.3761226733525594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,1,0.374944011370341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,7,0.42104001839955646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,3,0.39291199048360187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,15,0.5085279941558838
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,7,0.41065601507822674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,3,0.39711999893188477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,15,0.5005066792170206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,31,0.5135306517283121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,31,0.5155786673227946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,63,0.5195093154907227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,float16,127,0.6416426499684652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,63,0.5206453402837118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,32,128,1,float16,fp8,127,0.6406879822413126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,1,0.7776319980621338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,1,0.7545866966247559
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,3,0.7793707052866617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,3,0.7979946931203207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,7,0.8139146963755289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,7,0.8362720012664795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,15,0.9941120147705078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,15,1.010581334431966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,31,1.0202826658884685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,1,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,31,1.0240000089009602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,1,0.014869333555301031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,3,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,float16,63,1.0304160118103027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,3,0.015706667055686314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,15,0.01674666628241539
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,7,0.015599999576807022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,7,0.01540800059835116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,15,0.01695999999841054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,32,128,1,float16,fp8,63,1.0340800285339355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,31,0.019952000429232914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,63,0.02019199977318446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,63,0.019978666057189304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,127,0.020224000016848247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,31,0.02027733375628789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,127,0.020400000115235645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,255,0.02420266717672348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,255,0.024005333582560223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,511,0.035690667728583016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,511,0.04037333279848099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,1023,0.062277331948280334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,1023,0.07478400071461995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,2047,0.13614933689435324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,2047,0.11496532956759135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,4095,0.2572266658147176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,8191,0.5006933212280273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,4095,0.2134880026181539
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,8191,0.4071306784947713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,float16,16383,1.003941297531128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,1,0.014229333649079004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,32,128,1,float16,fp8,16383,0.8022720019022623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,1,0.014463999619086584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,3,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,3,0.014362666755914688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,7,0.014864000181357065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,15,0.0161920003592968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,7,0.014474666366974512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,63,0.016255999604860943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,15,0.016261332978804905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,31,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,31,0.016506666938463848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,63,0.016410666207472484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,127,0.019199999670187633
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,127,0.019050666441520054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,255,0.027327999472618103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,255,0.03190933416287104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,511,0.06289066871007283
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,511,0.049413333336512245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,1023,0.11588266491889954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,2047,0.2188640038172404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,1023,0.09162132938702901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,4095,0.4221973419189453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,2047,0.18145600954691568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,4095,0.346288005510966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,3,0.009423999736706415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,1,0.009525333220760027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,fp8,8191,0.688437302907308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,24,24,128,1,float16,float16,8191,0.8300960063934326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,1,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,3,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,7,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,7,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,15,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,15,0.00955200009047985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,31,0.010144000252087912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,31,0.010298666854699453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,63,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,255,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,63,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,127,0.011877333124478659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,127,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,255,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,511,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,511,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,1023,0.017450666675964992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,1023,0.017727999637524288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,4095,0.03714666763941447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,2047,0.017829333742459614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,2047,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,4095,0.037274666130542755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,8191,0.05365333457787832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,16383,0.09487467010815938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,8191,0.05128533144791921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,16383,0.08508800466855367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,float16,32767,0.16568533579508463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,1,0.009359999870260557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,24,24,128,1,float16,fp8,32767,0.15053866306940714
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,1,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,3,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,3,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,15,0.00973866693675518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,15,0.009888000165422758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,7,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,63,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,31,0.010314666976531347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,31,0.010378666842977205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,63,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,127,0.012063999970753988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,127,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,255,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,511,0.016650666793187458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,255,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,511,0.016575999557971954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,1023,0.02957333376010259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,2047,0.037263999382654824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,1023,0.029306667546431225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,2047,0.036933332681655884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,4095,0.051407997806866966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,4095,0.054234668612480164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,8191,0.09570133686065674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,16383,0.1511520047982534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,8191,0.08539733290672302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,16383,0.16665599743525186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,float16,32767,0.30818132559458417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,1,0.02049066623051961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,24,24,128,1,float16,fp8,32767,0.2762719988822937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,1,0.02102400114138921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,3,0.021690666675567627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,3,0.02142400046189626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,15,0.027765333652496338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,7,0.02181866765022278
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,7,0.022592000663280487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,15,0.027503999571005504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,31,0.027653334041436512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,127,0.03678400069475174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,31,0.02752000093460083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,63,0.0278613343834877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,127,0.03294399877389272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,63,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,255,0.056090667843818665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,511,0.08865066369374593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,255,0.053530668218930565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,511,0.10621866583824158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,1023,0.16723199685414633
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,1023,0.19910933574040732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,2047,0.3808639844258626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,4095,0.6181919972101847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,fp8,2047,0.31876800457636517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,24,24,128,1,float16,float16,4095,0.7461547056833903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,3,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,1,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,1,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,3,0.009946666657924652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,7,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,7,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,15,0.009914666414260864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,15,0.010133333504199982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,31,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,63,0.012202666451533636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,63,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,127,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,255,0.012437333663304647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,127,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,255,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,511,0.02790933350721995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,511,0.027834666272004444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,1023,0.03209600100914637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,2047,0.046207999189694725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,1023,0.032399999598662056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,2047,0.04366933306058248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,4095,0.07711466650168101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,4095,0.06959466636180878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,8191,0.1200320025285085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,8191,0.13179733355840048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,16383,0.21739733219146729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,16383,0.2388533353805542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,fp8,32767,0.4086506764094035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,24,24,128,1,float16,float16,32767,0.4536213477452596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,1,0.03737066686153412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,1,0.03830400109291077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,7,0.04010133445262909
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,3,0.03885866701602936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,3,0.03979199876387914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,7,0.040735999743143715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,15,0.050069332122802734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,15,0.05045866469542185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,31,0.05083733300367991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,63,0.05609600245952606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,31,0.05049600203831991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,63,0.050800000627835594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,127,0.06838933130105336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,255,0.1035146713256836
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,127,0.06698133548100789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,255,0.10082667072614034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,511,0.1678239901860555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,511,0.1994346578915914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,1023,0.38591468334198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,1023,0.31985066334406537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,fp8,2047,0.6128960053126017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,24,24,128,1,float16,float16,2047,0.7499573230743408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,3,0.07252266506354015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,1,0.06968533496061961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,1,0.0713919997215271
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,3,0.07435200115044911
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,7,0.07498133182525635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,15,0.09457600116729736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,31,0.10211732983589172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,7,0.07671999931335449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,15,0.09514666597048442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,31,0.09559999903043111
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,63,0.10381333033243816
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,63,0.1034346620241801
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,255,0.19504533211390176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,127,0.12661866346995035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,127,0.1279306709766388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,255,0.19163199265797934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,float16,511,0.3763306538263957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,24,24,128,1,float16,fp8,511,0.31996800502141315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,1,0.1346506675084432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,3,0.13948266704877219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,1,0.1386240025361379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,3,0.1430239975452423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,7,0.15019200245539346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,7,0.14934933185577393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,15,0.1919999917348226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,15,0.18811732530593872
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,31,0.19739200671513876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,31,0.19734400510787964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,63,0.19985600312550864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,63,0.2002453406651815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,127,0.2464746634165446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,1,0.008826666822036108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,127,0.24513065814971924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,float16,255,0.3786933422088623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,3,0.009039999917149544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,24,24,128,1,float16,fp8,255,0.37412798404693604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,1,0.009136000027259191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,3,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,7,0.009205333267649015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,7,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,15,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,31,0.011584000041087469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,31,0.011637333780527115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,63,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,127,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,63,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,255,0.013455999394257864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,255,0.013493333011865616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,127,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,511,0.019237333287795384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,2047,0.05363733569780985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,1023,0.030063999195893604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,1023,0.030943999687830608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,511,0.019098666807015736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,2047,0.06250133117039998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,4095,0.1147093375523885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,4095,0.10954667131106059
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,8191,0.20579200983047485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,16383,0.4272960027058919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,8191,0.21821333964665732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,16383,0.39930133024851483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,fp8,32767,1.1977226734161377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,24,24,128,1,float16,float16,32767,1.1835040251413982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,1,0.2683200041453044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,3,0.2932373285293579
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,1,0.2717439929644267
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,3,0.2862933278083801
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,7,0.3099466760953267
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,7,0.31461866696675617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,15,0.37744001547495526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,15,0.38310933113098145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,31,0.38729600111643475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,63,0.39202133814493817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,31,0.38849600156148273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,63,0.39154668649037677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,float16,127,0.48389867941538495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,24,24,128,1,float16,fp8,127,0.4824693202972412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,1,0.5683519840240479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,1,0.5851840178171793
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,3,0.600272019704183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,3,0.5865546862284342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,7,0.6133333444595337
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,7,0.6304266850153605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,15,0.7478773593902588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,15,0.7598133087158203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,1,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,1,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,31,0.7669653097788492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,3,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,7,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,31,0.7700160344441732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,3,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,float16,63,0.7744426727294922
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,24,24,128,1,float16,fp8,63,0.7770453294118246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,7,0.015322666615247726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,31,0.016879999389251072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,15,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,15,0.015306666493415833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,31,0.01706133286158244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,63,0.017055999487638474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,63,0.017263999829689663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,255,0.020021333048741024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,127,0.01732800031701724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,127,0.01747200017174085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,511,0.03249066571394602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,255,0.020351999749739964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,1023,0.05099200208981832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,2047,0.11414933204650879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,511,0.028394666810830433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,1023,0.06549333532651265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,2047,0.09222933650016785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,4095,0.21294933557510376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,8191,0.34585599104563397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,4095,0.18145600954691568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,8191,0.41201066970825195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,float16,16383,0.8292427062988281
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,24,24,128,1,float16,fp8,16383,0.6774453322092692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,1,0.015034666905800501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,1,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,3,0.015552000453074774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,3,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,15,0.016656000167131424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,7,0.015530666957298914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,7,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,15,0.01685333376129468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,31,0.019914666811625164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,31,0.020069333414236706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,63,0.020026666422684986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,127,0.020309332758188248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,63,0.020213333268960316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,127,0.02042666698495547
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,255,0.024351999163627625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,255,0.023973333338896435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,1023,0.062362665931383766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,511,0.04036800066630045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,1023,0.07402133444945018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,511,0.03588266670703888
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,2047,0.1362933317820231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,2047,0.1151626706123352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,8191,0.5019733508427938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,4095,0.2570506731669108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,4095,0.2134986718495687
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,float16,16383,0.9979093074798584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,8191,0.4081759850184123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,16,128,1,float16,fp8,16383,0.8053599993387858
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,1,0.009328000247478485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,3,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,1,0.009509333098928133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,7,0.009690666571259499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,7,0.009226666763424873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,3,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,15,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,31,0.010288000106811523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,15,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,31,0.0102186668664217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,63,0.011877333124478659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,63,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,255,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,127,0.011861333002646765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,127,0.011754666765530905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,255,0.012015999605258306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,511,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,511,0.016565332810084026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,1023,0.01736533393462499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,1023,0.017514667163292568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,2047,0.017738666385412216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,2047,0.017743999759356182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,4095,0.02959999938805898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,4095,0.02957333376010259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,8191,0.04062400013208389
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,16383,0.07121066749095917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,8191,0.04065066576004028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,32767,0.12090667088826497
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,16383,0.06181333462397257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,32767,0.1106719970703125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,float16,65535,0.21570666631062826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,3,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,1,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,16,128,1,float16,fp8,65535,0.19402666886647543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,3,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,7,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,31,0.010197333370645842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,15,0.009808000177145004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,15,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,7,0.009701333319147428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,63,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,63,0.01209066684047381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,255,0.012266666938861212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,127,0.011994666109482447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,255,0.012058666596810022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,511,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,1023,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,511,0.016442666451136272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,1023,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,2047,0.029765332738558452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,2047,0.029616000751654308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,4095,0.04028266668319702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,4095,0.04012800008058548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,8191,0.06640533109505971
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,8191,0.059215997656186424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,32767,0.17433599630991617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,16383,0.10046399633089702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,16383,0.10959999759991963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,32767,0.1947093407313029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,float16,65535,0.3667680025100708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,3,0.015466666469971338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,1,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,1,0.015429332852363586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,16,128,1,float16,fp8,65535,0.3274986743927002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,3,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,7,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,7,0.016234666109085083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,15,0.019541333119074505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,15,0.019679999599854153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,31,0.019621333728233974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,31,0.019744000087181728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,63,0.019765333582957584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,63,0.019765333582957584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,255,0.03921599934498469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,127,0.023711999257405598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,127,0.024010665714740753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,255,0.035631999373435974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,2047,0.25813867648442584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,511,0.07487999896208446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,511,0.061349332332611084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,1023,0.13664533694585165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,1023,0.11584533254305522
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,2047,0.21853333711624146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,4095,0.5019199848175049
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,4095,0.41977067788441974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,float16,8191,1.0470666885375977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,1,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,3,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,1,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,16,128,1,float16,fp8,8191,0.8385653495788574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,3,0.009973333527644476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,7,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,7,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,15,0.009722666814923286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,15,0.010010666524370512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,31,0.010351999973257383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,31,0.010421333213647207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,63,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,63,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,255,0.01250133290886879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,127,0.012138667205969492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,255,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,511,0.016837333639462788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,511,0.01681600014368693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,2047,0.037258667250474296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,1023,0.0295413335164388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,1023,0.029578665892283123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,4095,0.05659199754397074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,2047,0.0377866675456365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,16383,0.16950400670369467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,8191,0.0976106623808543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,8191,0.08789333701133728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,4095,0.05206400156021118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,16383,0.15245333313941956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,32767,0.3120693365732829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,32767,0.28032533327738446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,float16,65535,0.5968319972356161
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,1,0.02619733413060506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,1,0.026736001173655193
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,16,128,1,float16,fp8,65535,0.5317920049031576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,3,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,7,0.02792533238728841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,3,0.027658666173617046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,7,0.028533334533373516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,63,0.035717333356539406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,15,0.03523733218510946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,31,0.03538133452335993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,15,0.0351946676770846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,31,0.03548266738653183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,63,0.036687999963760376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,127,0.04781866570313772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,255,0.0721919983625412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,127,0.04338666796684265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,511,0.13780799508094788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,255,0.06986666719118755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,511,0.11505066355069478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,1023,0.26238399744033813
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,1023,0.21902932723363241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,2047,0.5041013161341349
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,2047,0.418938676516215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,fp8,4095,0.8137333393096924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,16,128,1,float16,float16,4095,0.9903679688771566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,1,0.048207998275756836
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,1,0.04961599906285604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,3,0.050186668833096824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,3,0.051589335004488625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,7,0.0518506666024526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,7,0.05300266544024149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,15,0.06509333352247874
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,15,0.06549866497516632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,31,0.06725333134333293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,31,0.0653653343518575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,63,0.07182399928569794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,63,0.06719466547171275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,255,0.1339306632677714
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,127,0.08667199810345967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,127,0.08781333764394124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,255,0.1311253309249878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,511,0.2606186668078105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,1,0.0916426678498586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,511,0.21958933273951212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,float16,1023,0.5096533298492432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,16,128,1,float16,fp8,1023,0.420687993367513
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,1,0.09371200203895569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,3,0.09480533003807068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,7,0.10095466176668803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,3,0.09742933511734009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,7,0.09871466954549153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,15,0.12763733665148416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,31,0.13388267159461975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,15,0.1251520017782847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,63,0.13555199901262918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,63,0.1360160013039907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,31,0.1295146644115448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,127,0.16733866930007935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,127,0.16607466340065002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,255,0.2564479907353719
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,511,0.4236319859822591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,fp8,255,0.25273066759109497
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,16,128,1,float16,float16,511,0.49694399038950604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,7,0.010010666524370512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,1,0.009935999910036722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,3,0.00961599995692571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,3,0.010010666524370512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,7,0.009626666704813639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,15,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,31,0.010666667173306147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,15,0.0100853331387043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,31,0.010640000303586325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,63,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,63,0.012191999703645706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,127,0.012442667037248611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,255,0.01259200026591619
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,127,0.01239466667175293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,255,0.012928000340859095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,511,0.028346667687098186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,2047,0.04442666471004486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,511,0.02815466622511546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,1023,0.03283733377854029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,1023,0.032986665765444435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,2047,0.04882133503754934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,4095,0.0860746701558431
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,8191,0.12410133083661397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,4095,0.07204799850781758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,8191,0.146506667137146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,16383,0.22209066152572632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,16383,0.2678133249282837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,32767,0.5109279950459799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,32767,0.418069322903951
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,float16,65535,1.007754643758138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,16,128,1,float16,fp8,65535,0.8296266396840414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,1,0.17778132359186807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,1,0.1827253301938375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,3,0.1869493325551351
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,3,0.18913600842158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,7,0.20618667205174765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,7,0.19970667362213135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,15,0.2537386616071065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,15,0.2554080088933309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,31,0.26054932673772174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,31,0.26126400629679364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,63,0.26365866263707477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,63,0.26394667228062946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,127,0.3252799908320109
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,127,0.3244160016377767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,fp8,255,0.49593599637349445
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,16,128,1,float16,float16,255,0.5003413359324137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,1,0.3767999807993571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,1,0.3752426703770955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,3,0.39325865109761554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,3,0.398965318997701
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,7,0.41122666994730633
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,7,0.4225546518961589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,15,0.5010826587677002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,15,0.5088160037994385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,1,0.009045333291093508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,31,0.5137653350830078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,31,0.5155733426411947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,1,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,63,0.519434650739034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,3,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,63,0.520634651184082
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,float16,127,0.6415679852167765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,16,128,1,float16,fp8,127,0.6411626736323038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,3,0.009610666582981745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,15,0.010293333480755487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,7,0.009450666606426239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,7,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,31,0.011621333658695221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,15,0.010037333394090334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,127,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,31,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,63,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,63,0.012015999605258306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,127,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,255,0.013690666606028875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,511,0.019333332777023315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,255,0.013872000078360239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,511,0.019391999890406925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,1023,0.04001066585381826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,1023,0.030559999247392017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,2047,0.07157333195209503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,4095,0.1104159951210022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,2047,0.06051200131575266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,4095,0.13306132952372232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,8191,0.25566933552424115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,16383,0.5055573383967081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,8191,0.2085813283920288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,16383,0.4094506502151489
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,float16,32767,1.266767978668213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,1,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,16,128,1,float16,fp8,32767,1.2369279861450195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,1,0.015168000012636185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,3,0.015504000087579092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,3,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,7,0.015034666905800501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,7,0.015365333606799444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,31,0.016832000265518825
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,15,0.015168000012636185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,15,0.015066667149464289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,63,0.017055999487638474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,31,0.017029333859682083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,63,0.01732800031701724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,127,0.017263999829689663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,255,0.0201706662774086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,127,0.01738133281469345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,511,0.028384000062942505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,255,0.02033599962790807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,511,0.03266666581233343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,1023,0.05091199775536855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,1023,0.06260266900062561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,2047,0.0930613378683726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,2047,0.11403200030326843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,4095,0.18097599347432455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,4095,0.212501327196757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,8191,0.4107999801635742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,8191,0.34613335132598877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,1,0.009098666409651438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,1,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,3,0.00922133338948091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,float16,16383,0.8403147061665853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,12,12,128,1,float16,fp8,16383,0.6760479609171549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,3,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,7,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,7,0.00956266683836778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,15,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,31,0.010053333515922228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,15,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,31,0.010330666477481524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,63,0.011920000116030375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,63,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,127,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,127,0.011695999652147293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,255,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,255,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,511,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,511,0.016271999726692837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,1023,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,1023,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,4095,0.017738666385412216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,2047,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,2047,0.017621333400408428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,4095,0.01770666614174843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,8191,0.02189333240191142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,8191,0.021514666577180225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,16383,0.030213333666324615
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,16383,0.028970666229724884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,32767,0.061093335350354515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,32767,0.04772266745567322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,float16,65535,0.10673600435256958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,12,12,128,1,float16,fp8,65535,0.08774933218955994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,1,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,1,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,7,0.009317333499590555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,3,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,3,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,15,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,15,0.009525333220760027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,7,0.009450666606426239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,63,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,31,0.010128000130256018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,63,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,127,0.011968000481526056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,31,0.01028266673286756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,127,0.011776000261306763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,255,0.012202666451533636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,511,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,255,0.01190399999419848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,1023,0.017514667163292568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,2047,0.017722666263580322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,511,0.016549333930015564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,1023,0.017632000148296356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,2047,0.017765333255132038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,4095,0.037274666130542755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,4095,0.03718400001525879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,8191,0.054010664423306785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,8191,0.0516480008761088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,16383,0.09490666786829631
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,16383,0.08532266815503438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,32767,0.16564800341924033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,32767,0.1506666640440623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,fp8,65535,0.27803200483322144
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,12,12,128,1,float16,float16,65535,0.3091786702473958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,3,0.014298666268587112
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,1,0.014277332772811254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,1,0.0143306665122509
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,3,0.014618666221698126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,7,0.01444799949725469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,7,0.014597332725922266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,31,0.01639466608564059
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,15,0.016063999384641647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,15,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,31,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,63,0.01659199967980385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,63,0.01647466669480006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,255,0.03151999910672506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,127,0.019007999449968338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,127,0.019189332922299702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,255,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,511,0.0631520003080368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,2047,0.21876800060272217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,511,0.04933333396911621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,1023,0.11609066526095073
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,1023,0.09177600344022115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,2047,0.18145600954691568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,4095,0.42239999771118164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,4095,0.34703465302785236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,fp8,8191,0.6910400390625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,1,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,1,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,12,12,128,1,float16,float16,8191,0.831397294998169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,3,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,7,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,3,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,7,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,15,0.009626666704813639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,15,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,31,0.010309333602587381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,63,0.01209066684047381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,63,0.012240000069141388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,255,0.012298667182525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,127,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,127,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,1023,0.029525332152843475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,511,0.016805333395799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,511,0.016741332908471424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,1023,0.02945599953333537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,255,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,2047,0.03712533414363861
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,4095,0.0539626677831014
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,2047,0.03687999894221624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,4095,0.05148266752560934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,8191,0.0951039989789327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,8191,0.08501332998275757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,16383,0.16622933745384216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,16383,0.150736004114151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,32767,0.30830933650334674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,32767,0.27542932828267414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,float16,65535,0.5935360193252563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,12,12,128,1,float16,fp8,65535,0.526528000831604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,1,0.020741333564122517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,7,0.021903999149799347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,1,0.021146667500336964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,3,0.02143466720978419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,3,0.021690666675567627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,31,0.02777066578467687
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,7,0.022410665949185688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,15,0.027493332823117573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,15,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,31,0.027722666660944622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,63,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,63,0.027829334139823914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,127,0.03721066564321518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,127,0.03305600086847941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,255,0.05600533386071523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,511,0.10626133282979329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,255,0.05352533360322317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,511,0.08920533458391826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,1023,0.19911466042200723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,2047,0.31932266553243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,1023,0.16774400075276694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,2047,0.3808053334554036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,3,0.03979199876387914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,fp8,4095,0.6193013191223145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,12,12,128,1,float16,float16,4095,0.7470293045043945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,1,0.03743999948104223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,1,0.03821333249409994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,3,0.03886399914820989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,7,0.040336000422636666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,7,0.04082666585842768
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,15,0.05019199848175049
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,15,0.05064000189304352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,31,0.05087999999523163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,63,0.05604266623655955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,127,0.06825066606203715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,31,0.05060799916585287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,127,0.06694399813810985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,255,0.10355200370152791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,63,0.05093333125114441
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,255,0.10083733002344768
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,511,0.1993866761525472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,511,0.16800532738367716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,float16,1023,0.3863893349965413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,1,0.07156800230344136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,12,12,128,1,float16,fp8,1023,0.3208799958229065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,1,0.06974400083223979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,3,0.07459733386834462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,3,0.07258133093516032
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,7,0.07484266658624013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,7,0.0766133318344752
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,15,0.09505599737167358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,15,0.09484799702962239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,63,0.10428800185521443
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,31,0.09573866923650105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,31,0.10211732983589172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,63,0.10357866684595744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,127,0.12758933504422507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,255,0.19565866390864053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,127,0.12687466541926065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,float16,511,0.37694935003916424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,255,0.19197867314020792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,1,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,12,12,128,1,float16,fp8,511,0.32070932785669964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,1,0.009999999776482582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,7,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,3,0.00997866690158844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,3,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,7,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,15,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,15,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,31,0.010506667196750641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,63,0.012229333321253458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,31,0.010687999427318573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,63,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,127,0.0122079998254776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,127,0.012272000312805176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,255,0.012671999633312225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,511,0.027690666417280834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,255,0.012250666817029318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,511,0.027813332776228588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,1023,0.03236266722281774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,1023,0.03242133309443792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,4095,0.06925333539644878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,2047,0.043840001026789345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,4095,0.0769760012626648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,8191,0.1320373316605886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,2047,0.045610666275024414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,8191,0.12004799644152324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,16383,0.21708800395329794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,16383,0.2392373283704122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,32767,0.45238399505615234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,32767,0.4084213177363078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,float16,65535,0.8842666943868002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,1,0.13436266779899597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,12,12,128,1,float16,fp8,65535,0.7974293231964111
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,1,0.1386240025361379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,3,0.13961066802342734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,3,0.1430186629295349
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,7,0.15025599797566733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,7,0.14933866262435913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,15,0.19218667348225912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,15,0.18811200062433878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,31,0.19739733139673868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,63,0.20040533939997354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,31,0.19747199614842734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,63,0.20012799898783365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,127,0.2465653419494629
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,127,0.24542399247487387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,float16,255,0.37911999225616455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,12,12,128,1,float16,fp8,255,0.3742080132166545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,1,0.26924800872802734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,1,0.2719680070877075
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,7,0.310208002726237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,3,0.2930399974187215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,3,0.2882399956385295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,7,0.31540799140930176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,15,0.3774133523305257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,15,0.3832480112711589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,31,0.3885013262430827
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,31,0.3870026667912801
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,63,0.39131732781728107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,63,0.39219733079274494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,float16,127,0.48447465896606445
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,1,0.00895999992887179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,3,0.008912000184257826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,1,0.00919999989370505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,7,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,3,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,12,12,128,1,float16,fp8,127,0.4827839930852254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,7,0.009423999736706415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,15,0.009941333283980688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,31,0.0116799995303154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,63,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,31,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,15,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,63,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,127,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,511,0.019023999571800232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,255,0.013679999858140945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,255,0.013749333719412485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,511,0.019029332945744198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,1023,0.03029866764942805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,1023,0.031093334158261616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,4095,0.11411199967066447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,2047,0.06227200229962667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,4095,0.1086346705754598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,2047,0.05315199991067251
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,8191,0.21760533253351846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,8191,0.20520534118016562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,1,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,16383,0.42741334438323975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,16383,0.3988000154495239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,float16,32767,1.1839306354522705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,1,0.009248000259200731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,12,12,128,1,float16,fp8,32767,1.1982293128967285
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,3,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,7,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,3,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,15,0.01007466639081637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,7,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,15,0.009999999776482582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,31,0.011616000284751257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,31,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,63,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,63,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,127,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,127,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,1023,0.04061333338419596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,255,0.01392000044385592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,255,0.013722666849692663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,511,0.01937599976857503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,511,0.019343999524911244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,1023,0.030394665896892548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,2047,0.06050133208433787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,4095,0.13310399651527405
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,2047,0.07201600074768066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,4095,0.11026133100191753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,8191,0.2557813326517741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,8191,0.2082186738650004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,16383,0.5085119803746542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,1,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,16383,0.4057653347651164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,float16,32767,1.2740320364634197
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,8,128,1,float16,fp8,32767,1.2245333194732666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,1,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,3,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,3,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,7,0.009125333279371262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,7,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,15,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,31,0.0102186668664217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,15,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,31,0.010144000252087912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,63,0.011829332758982977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,63,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,127,0.011877333124478659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,255,0.011861333002646765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,127,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,255,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,511,0.01611199975013733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,511,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,1023,0.017258666455745697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,1023,0.017221332838137943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,2047,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,2047,0.01740266631046931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,4095,0.017653333644072216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,8191,0.029461334149042766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,4095,0.01759999990463257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,8191,0.029616000751654308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,32767,0.07179200152556102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,16383,0.040607998768488564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,16383,0.04031999905904134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,32767,0.06270933151245117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,65535,0.12333333492279053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,65535,0.11338667074839275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,1,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,3,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,float16,131071,0.22778133551279703
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,8,128,1,float16,fp8,131071,0.2068693240483602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,15,0.009269333134094873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,7,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,3,0.009418666362762451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,7,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,15,0.009610666582981745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,31,0.010079999764760336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,63,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,63,0.011962667107582092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,31,0.010384000216921171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,127,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,511,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,127,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,511,0.016309333344300587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,255,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,255,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,1023,0.01740266631046931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,1023,0.0174346665541331
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,2047,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,2047,0.017738666385412216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,4095,0.029733332494894665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,4095,0.029482667644818623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,8191,0.04058666775623957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,8191,0.04067733387152354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,16383,0.07108800113201141
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,32767,0.12101333340009053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,16383,0.06155733267466227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,32767,0.11105599999427795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,65535,0.2147093415260315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,1,0.015541333705186844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,65535,0.19337066014607748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,1,0.014906667172908783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,3,0.015610666324694952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,3,0.015450666348139444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,fp8,131071,0.36187732219696045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,8,128,1,float16,float16,131071,0.40720534324645996
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,7,0.015610666324694952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,31,0.019920000185569126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,7,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,15,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,15,0.017114666601022083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,31,0.02027200038234393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,63,0.020181333025296528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,63,0.02027733375628789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,127,0.02027200038234393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,127,0.02041600023706754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,255,0.024512000381946564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,255,0.023930666347344715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,511,0.04058666775623957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,511,0.035674666364987694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,1023,0.07491733133792877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,1023,0.062208001812299095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,2047,0.11500266194343567
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,2047,0.13637333114941916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,4095,0.2579093376795451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,1,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,4095,0.21414399147033691
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,1,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,8191,0.5012799898783366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,8191,0.4081439971923828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,fp8,16383,0.7993653615315756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,8,128,1,float16,float16,16383,0.9983946482340494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,7,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,3,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,3,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,7,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,15,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,15,0.009989333028594652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,31,0.010154666379094124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,31,0.01051733394463857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,63,0.012042666474978128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,63,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,127,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,127,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,255,0.01246400053302447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,1023,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,255,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,511,0.016469333320856094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,1023,0.017743999759356182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,511,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,2047,0.029711998999118805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,4095,0.04027733455101649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,2047,0.029781334102153778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,4095,0.04055999964475632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,8191,0.05937600135803223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,16383,0.10970667004585266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,8191,0.06676266590754192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,16383,0.10006933410962422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,32767,0.1938719948132833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,32767,0.17477333545684814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,65535,0.3665173451105754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,65535,0.32808534304300946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,1,0.014949332922697067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,1,0.015173333386580149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,float16,131071,0.7130453586578369
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,3,0.015605332950750986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,8,128,1,float16,fp8,131071,0.6397013266881307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,3,0.015295999745527903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,7,0.01607999950647354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,31,0.019632000476121902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,15,0.01952533299724261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,31,0.019679999599854153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,63,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,7,0.016293333222468693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,15,0.019626667102177937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,63,0.019871999820073444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,127,0.02348266790310542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,127,0.024453334510326385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,255,0.03922666609287262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,511,0.07467199862003326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,255,0.03537066777547201
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,511,0.061253334085146584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,1023,0.13634133338928223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,1023,0.11582400401433308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,2047,0.2580373287200928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,2047,0.21893332401911417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,4095,0.5032906532287598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,4095,0.41946132977803546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,1,0.026330667237440746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,3,0.02717866748571396
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,1,0.026767998933792114
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,float16,8191,1.0481493473052979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,8,128,1,float16,fp8,8191,0.8358453114827474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,3,0.02773333340883255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,15,0.03522666543722153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,7,0.02807466685771942
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,7,0.028533334533373516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,15,0.03514666606982549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,31,0.03543466577927271
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,31,0.035386666655540466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,63,0.03669333209594091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,63,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,127,0.047872001926104225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,255,0.0719893326361974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,127,0.04343999922275543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,255,0.0698880006869634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,511,0.13834666212399802
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,511,0.11512533823649089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,1023,0.26209600766499835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,float16,2047,0.5056000153223673
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,1023,0.21925334135691324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,8,128,1,float16,fp8,2047,0.41861867904663086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,1,0.04827733337879181
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,1,0.049738665421803795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,3,0.050053333242734276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,3,0.05146666864554087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,15,0.06508266429106395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,7,0.051967998345692955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,7,0.053317333261171974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,31,0.0674239993095398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,127,0.08796266714731853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,31,0.0655680000782013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,63,0.07172266642252605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,15,0.06549866497516632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,63,0.06789333124955495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,127,0.08709866801897685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,255,0.1337493360042572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,511,0.2602826754252116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,255,0.13117333253224692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,1,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,511,0.21968533595403036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,1,0.00985599992175897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,3,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,float16,1023,0.5095359881718954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,8,128,1,float16,fp8,1023,0.42048001289367676
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,31,0.010288000106811523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,7,0.00956266683836778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,7,0.009642666826645533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,15,0.010015999898314476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,15,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,63,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,31,0.01051733394463857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,63,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,127,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,127,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,2047,0.03772799919048945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,255,0.01250133290886879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,511,0.017008000363906223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,255,0.01227733368674914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,511,0.016751999656359356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,1023,0.029658667743206024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,1023,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,4095,0.05699733396371206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,2047,0.03737599899371465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,8191,0.09737599889437358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,8191,0.08756800492604573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,4095,0.05216533442338308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,16383,0.16922134160995483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,16383,0.15250133474667868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,32767,0.3116640051205953
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,32767,0.28084800640741986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,65535,0.5969386498133341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,65535,0.5319039821624756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,1,0.09151466687520345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,3,0.09758933385213216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,1,0.09387200077374776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,3,0.09473066528638203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,float16,131071,1.0112533569335938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,8,128,1,float16,fp8,131071,0.8276800314585367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,15,0.12518399953842163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,7,0.09884267052014668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,31,0.1295840044816335
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,7,0.10096533099810283
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,15,0.1278986632823944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,31,0.1339413324991862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,63,0.13590400417645773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,63,0.13632532954216003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,127,0.16775999466578165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,127,0.16614400347073874
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,255,0.2566346724828084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,255,0.2530133326848348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,float16,511,0.4987146854400635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,8,128,1,float16,fp8,511,0.42366401354471844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,1,0.17769600947697958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,1,0.18277867635091147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,3,0.1864159901936849
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,7,0.20666666825612387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,3,0.1890986760457357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,7,0.20018666982650757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,15,0.2544800043106079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,15,0.2553386688232422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,31,0.2603093385696411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,31,0.2612053354581197
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,63,0.26387200752894086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,63,0.26419200499852497
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,1,0.0100853331387043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,127,0.32600533962249756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,1,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,3,0.009706666693091393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,127,0.3243040045102437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,3,0.009973333527644476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,float16,255,0.5011306603749593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,7,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,15,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,7,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,15,0.010128000130256018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,8,128,1,float16,fp8,255,0.4964053233464559
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,31,0.01073066641887029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,31,0.010666667173306147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,255,0.012730666746695837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,63,0.012426666915416718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,63,0.012357333054145178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,127,0.012330666184425354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,127,0.012341332932313284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,255,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,511,0.028325334191322327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,511,0.028186666468779247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,1023,0.03316266586383184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,2047,0.04978133241335551
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,2047,0.044495999813079834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,4095,0.08610666791598003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,1023,0.03288000077009201
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,8191,0.14703466494878134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,4095,0.0726986676454544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,16383,0.2675306598345439
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,8191,0.12461333473523457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,16383,0.2221440076828003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,32767,0.510261336962382
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,32767,0.4174613157908122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,1,0.009674666449427605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,float16,65535,1.0082080364227295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,1,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,8,128,1,float16,fp8,65535,0.8173120021820068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,3,0.010239999741315842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,3,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,7,0.009999999776482582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,7,0.009690666571259499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,15,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,15,0.009877333417534828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,31,0.010768000036478043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,31,0.010527999450763067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,63,0.012437333663304647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,255,0.012885333349307379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,127,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,127,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,63,0.012426666915416718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,255,0.012383999923865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,511,0.028277332584063213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,1023,0.032773333291212715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,511,0.028223998844623566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,1023,0.03314133236805598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,2047,0.04453866680463155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,4095,0.08117866516113281
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,2047,0.04916266600290934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,8191,0.1225493351618449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,4095,0.07258133093516032
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,8191,0.14666133125623068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,16383,0.2674293319384257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,16383,0.22204800446828207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,32767,0.5099039872487386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,32767,0.4175573190053304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,1,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,1,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,fp8,65535,0.8188266754150391
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,4,128,1,float16,float16,65535,1.0064853032430012
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,3,0.008976000050703684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,3,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,7,0.009514666472872099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,7,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,15,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,31,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,15,0.009503999724984169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,63,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,31,0.010170666500926018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,63,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,127,0.01190399999419848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,255,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,127,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,255,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,511,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,1023,0.017301333447297413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,511,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,2047,0.017231999586025875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,1023,0.017360000560681026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,2047,0.01749333366751671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,4095,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,4095,0.0173333336909612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,8191,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,16383,0.01758933315674464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,8191,0.017477333545684814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,16383,0.018063999712467194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,32767,0.02372266600529353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,32767,0.02327466756105423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,65535,0.04463466505209605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,65535,0.03422933320204417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,1,0.00901333304742972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,1,0.009354666496316591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,3,0.008997333546479544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,fp8,131071,0.06548800071080525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,3,0.009423999736706415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,4,128,1,float16,float16,131071,0.07816533247629802
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,7,0.009061333412925402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,7,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,15,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,15,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,31,0.010239999741315842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,127,0.011770666887362799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,31,0.010170666500926018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,63,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,127,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,63,0.011823999385039011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,511,0.01599466676513354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,255,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,255,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,511,0.016106666376193363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,1023,0.017407999684413273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,2047,0.017488000293572743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,1023,0.017573333034912746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,2047,0.01735466718673706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,4095,0.01754133279124896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,8191,0.029690665503342945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,4095,0.017610666652520496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,8191,0.029418667157491047
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,16383,0.04088533421357473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,16383,0.040549332896868386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,32767,0.07187733550866444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,32767,0.06208533545335134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,65535,0.1237386663754781
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,65535,0.11271466811498006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,1,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,1,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,float16,131071,0.22753065824508667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,4,128,1,float16,fp8,131071,0.20578134059906006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,3,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,7,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,3,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,7,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,63,0.012063999970753988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,15,0.009999999776482582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,15,0.010106666634480158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,31,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,31,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,63,0.011930666863918304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,127,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,255,0.013877333452304205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,127,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,1023,0.040778666734695435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,511,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,255,0.013770667215188345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,511,0.019194666296243668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,1023,0.03053866575161616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,2047,0.07152533531188965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,2047,0.06038933495680491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,4095,0.11081066727638245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,4095,0.1330933372179667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,8191,0.25566399097442627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,8191,0.20891199509302774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,16383,0.5069973468780518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,16383,0.4049706856409709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,1,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,1,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,fp8,32767,1.2229866981506348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,3,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,4,128,1,float16,float16,32767,1.2799839973449707
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,3,0.00938666673998038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,7,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,15,0.009370666618148485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,7,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,15,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,127,0.011866666376590729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,31,0.010005333150426546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,63,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,31,0.010314666976531347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,63,0.011877333124478659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,127,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,255,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,511,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,1023,0.017360000560681026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,511,0.016410666207472484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,255,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,2047,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,2047,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,1023,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,4095,0.029850666721661884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,4095,0.029552000264326733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,8191,0.04070399949947993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,8191,0.04054400076468786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,16383,0.06169599791367849
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,16383,0.07050133248170216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,32767,0.1207360029220581
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,32767,0.1109279990196228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,65535,0.21572800477345785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,65535,0.19382933775583902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,float16,131071,0.4068959951400757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,1,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,1,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,3,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,4,128,1,float16,fp8,131071,0.36101865768432617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,3,0.015626666446526844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,7,0.015594666202863058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,7,0.015765332927306492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,15,0.01664000004529953
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,15,0.016879999389251072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,31,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,31,0.020138667275508244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,127,0.0204373337328434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,63,0.020069333414236706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,63,0.020309332758188248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,127,0.020266667008399963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,255,0.024506665766239166
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,255,0.023887999355793
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,511,0.040207999447981514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,1023,0.07498666644096375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,511,0.035717333356539406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,1023,0.06223999957243601
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,4095,0.25814932584762573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,2047,0.11572266618410747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,2047,0.1365013321240743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,1,0.014959999670584997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,4095,0.21429866552352905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,8191,0.4080746571222941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,8191,0.5009546677271525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,1,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,3,0.01544533297419548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,float16,16383,0.9959999720255533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,3,0.01552533358335495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,4,128,1,float16,fp8,16383,0.7977813084920248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,7,0.015882667154073715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,7,0.016085332880417507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,15,0.019589333484570186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,15,0.01966933285196622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,63,0.01974933346112569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,31,0.019653332730134327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,31,0.019632000476121902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,63,0.01972266659140587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,127,0.023562667270501454
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,127,0.024170666933059692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,511,0.07482133309046428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,255,0.0391146664818128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,255,0.03544000039498011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,511,0.060959999759991966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,1023,0.13641066352526346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,1023,0.11590933799743652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,2047,0.25913600126902264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,2047,0.21861332654953003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,float16,4095,0.5036160151163737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,1,0.026074667771657307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,4,128,1,float16,fp8,4095,0.4208373228708903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,1,0.026848000784715016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,3,0.02699200063943863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,7,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,3,0.027642667293548584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,15,0.035173334181308746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,7,0.02845333268245061
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,15,0.03536533315976461
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,31,0.03565866748491923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,31,0.035536001125971474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,63,0.036602665980656944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,63,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,127,0.04320000112056732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,127,0.0480373352766037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,255,0.07260799904664357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,255,0.06982400019963582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,511,0.1381013294061025
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,511,0.11555199821790059
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,1023,0.21925334135691324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,2047,0.5049120187759399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,float16,1023,0.26206932465235394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,1,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,1,0.009882666791478792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,4,128,1,float16,fp8,2047,0.41918400923411053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,3,0.009418666362762451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,7,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,7,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,3,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,15,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,15,0.00979200005531311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,31,0.010469333579142889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,31,0.010330666477481524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,63,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,63,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,127,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,255,0.012330666184425354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,255,0.012037333101034164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,1023,0.017658667018016178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,511,0.01634666696190834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,2047,0.029743999242782593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,511,0.016730666160583496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,1023,0.017616000026464462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,2047,0.02961066613594691
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,4095,0.040234667559464775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,8191,0.06638399759928386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,4095,0.03999999910593033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,16383,0.10893332958221436
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,8191,0.05909866591294607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,32767,0.19504533211390176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,16383,0.10061867038408916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,32767,0.1746506690979004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,65535,0.3664586544036865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,65535,0.3267893393834432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,1,0.04831466575463613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,fp8,131071,0.6407093207041422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,1,0.04983466863632202
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,4,128,1,float16,float16,131071,0.7142186959584554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,3,0.05008000135421753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,3,0.05157866577307383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,7,0.05300800005594889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,7,0.05188799897829691
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,15,0.0654720018307368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,15,0.06491733094056447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,31,0.06714133421579997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,63,0.07211199899514516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,63,0.06729066868623097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,127,0.08815999825795491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,31,0.06557866434256236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,255,0.13452266653378805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,127,0.0869813362757365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,255,0.1309599975744883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,511,0.260426660378774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,511,0.2200373411178589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,float16,1023,0.5100160042444865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,1,0.09408000111579895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,4,128,1,float16,fp8,1023,0.42080533504486084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,1,0.09146133065223694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,3,0.09493333101272583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,3,0.0974079966545105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,7,0.0986400047938029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,7,0.1011946698029836
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,15,0.1276746690273285
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,15,0.12532800436019897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,31,0.13426666458447775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,63,0.13662933309872946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,31,0.12974933783213297
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,63,0.13613333304723105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,127,0.16771199305852255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,127,0.16658666729927063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,255,0.25728533665339154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,1,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,3,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,1,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,255,0.2532053391138713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,7,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,float16,511,0.497050682703654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,15,0.010048000141978264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,15,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,31,0.010591999938090643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,4,128,1,float16,fp8,511,0.4244106610616048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,31,0.010309333602587381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,63,0.012069333344697952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,127,0.012272000312805176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,127,0.01231466606259346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,255,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,255,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,511,0.01695999999841054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,511,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,1023,0.0296426663796107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,2047,0.03775999943415324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,8191,0.08807466427485149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,2047,0.03740799923737844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,1023,0.029648000995318096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,4095,0.05739733576774597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,8191,0.09670399626096089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,4095,0.052229334910710655
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,16383,0.16933866341908774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,32767,0.31166932980219525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,16383,0.15294399857521057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,32767,0.2802506685256958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,65535,0.5342026551564535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,1,0.046810666720072426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,1,0.047354668378829956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,fp8,131071,0.8276480038960775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,3,0.04857600231965383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,131071,1.0089386304219563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,4,128,1,float16,float16,65535,0.5961600144704183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,3,0.04901866614818573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,7,0.050586665670077004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,15,0.06322666505972545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,15,0.06262399752934773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,7,0.05052266518274943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,31,0.06357866525650024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,31,0.0626986672480901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,63,0.06309333443641663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,127,0.07634133100509644
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,127,0.07578133543332417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,255,0.11717866857846577
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,63,0.06374399860699971
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,255,0.11557867129643758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,511,0.1999839941660563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,1023,0.3649760087331136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,511,0.19883733987808228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,2047,0.6950986385345459
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,2047,0.6899360020955404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,1023,0.3625919818878174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,fp8,4095,1.343839963277181
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,1,0.04731200138727824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,1,0.04805333415667216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,3,0.048954665660858154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,3,0.049466664592425026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,7,0.05115200082461039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,7,0.05120533208052317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,1,128,1,float16,float16,4095,1.3572907447814941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,15,0.06356266637643178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,15,0.06348266700903575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,31,0.06423999865849812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,63,0.0639519989490509
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,31,0.06330133477846782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,63,0.06437866886456807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,127,0.07690133154392242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,255,0.1183786690235138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,255,0.11803733309110005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,511,0.20096532503763834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,511,0.2007146676381429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,127,0.07684800028800964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,1023,0.3643093506495158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,1023,0.36511464913686115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,2047,0.7001760005950928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,4095,1.3531840642293294
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,fp8,2047,0.691215991973877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,2,128,1,float16,float16,4095,1.3601279258728027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,3,0.04953599969546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,7,0.0513866643110911
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,1,0.04780266682306925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,15,0.06362666686375935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,31,0.06345599889755249
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,63,0.06401599943637848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,127,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,1,0.04738666613896688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,3,0.04915733138720194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,127,0.012970666090647379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,7,0.050986667474110924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,15,0.06374933322270711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,511,0.03982399900754293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,511,0.034688000877698265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,31,0.0640533318122228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,255,0.02610666553179423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,1023,0.05761066575845083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,2047,0.08243200182914734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,4095,0.10634133219718933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,4095,0.13607466220855713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,255,0.02143999934196472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,float16,63,0.0642133355140686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,1023,0.07361599802970886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,4,128,1,float16,fp8,2047,0.0981813371181488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,1,0.04798933366934458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,15,0.06364800035953522
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,3,0.04964800179004669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,7,0.05148266752560934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,31,0.06382933259010315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,63,0.06406400104363759
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,1,0.04741866886615753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,127,0.01616000011563301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,127,0.008736000085870424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,3,0.049269333481788635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,7,0.051221330960591636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,15,0.06369066735108693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,255,0.030847998956839245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,255,0.017792000124851864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,31,0.06418666740258534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,63,0.0645066648721695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,1023,0.06218666831652323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,2047,0.05906666815280914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,2047,0.08381866415341695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,4095,0.1200373371442159
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,1,0.009808000177145004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,4095,0.08683733145395915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,511,0.0317493329445521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,1,0.00961599995692571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,3,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,3,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,fp8,1023,0.04025600105524063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,128,8,128,1,float16,float16,511,0.047695999344189964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,7,0.009488000224033991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,31,0.010453333457310995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,7,0.00985599992175897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,15,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,31,0.010501333822806677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,15,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,63,0.012256000190973282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,127,0.012346666306257248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,127,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,255,0.012282667060693106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,255,0.012602667013804117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,511,0.027978666126728058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,511,0.028255999088287354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,63,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,1023,0.0327360009153684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,2047,0.04386133452256521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,1023,0.03271466741959254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,2047,0.04393066465854645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,4095,0.06572799881299336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,4095,0.06603733201821645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,16383,0.19569599628448486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,8191,0.10948266585667928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,float16,16383,0.19608000914255777
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,1,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,1,128,1,float16,fp8,8191,0.1090133289496104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,3,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,3,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,7,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,1,0.009925333162148794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,15,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,7,0.009797333429257074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,15,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,63,0.012191999703645706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,31,0.010629333555698395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,63,0.012389333297808966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,127,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,127,0.012565333396196365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,255,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,255,0.012543999900420507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,31,0.010341333225369453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,511,0.02824000020821889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,511,0.028170667588710785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,1023,0.03286933402220408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,1023,0.03272533416748047
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,2047,0.04398400088151296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,2047,0.043824002146720886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,4095,0.06586133440335591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,8191,0.1090666651725769
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,16383,0.19549334049224854
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,float16,16383,0.19637332359949747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,8191,0.10935999949773152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,3,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,2,128,1,float16,fp8,4095,0.06584000090758006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,1,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,3,0.009610666582981745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,1,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,7,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,15,0.009914666414260864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,31,0.010464000205198923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,63,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,63,0.012149333953857422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,127,0.012272000312805176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,255,0.012245333443085352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,255,0.012543999900420507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,7,0.009797333429257074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,1023,0.032586666444937386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,511,0.02812266598145167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,511,0.02802666773398717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,1023,0.0327360009153684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,2047,0.015498666713635126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,4095,0.018602666755517323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,8191,0.023071999351183575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,4095,0.01793066660563151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,2047,0.017711999515692394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,8191,0.02201066662867864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,fp8,16383,0.02646933247645696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,3,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,1,0.00955200009047985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,4,128,1,float16,float16,16383,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,1,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,15,0.009941333283980688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,7,0.009775999933481216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,15,0.009866666669646898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,63,0.012335999558369318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,31,0.010506667196750641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,7,0.009722666814923286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,63,0.01219733307758967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,3,0.00980266680320104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,127,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,255,0.012549333274364471
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,127,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,255,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,511,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,1023,0.010293333480755487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,1023,0.01358933374285698
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,4095,0.01340266689658165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,2047,0.015552000453074774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,511,0.027952000498771667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,2047,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,4095,0.016330666840076447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,8191,0.025631998976071674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,16383,0.022837333381175995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,fp8,8191,0.01850133389234543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,128,8,128,1,float16,float16,16383,0.03049066662788391
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,3,0.009258666386206945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,7,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,3,0.009066666786869368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,15,0.009839999799927076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,1,0.009125333279371262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,7,0.009328000247478485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,15,0.009989333028594652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,31,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,63,0.011551999797423681
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,31,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,127,0.011610666910807291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,63,0.011754666765530905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,255,0.013584000368913015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,255,0.013717333475748697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,511,0.01916266605257988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,511,0.019082666685183842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,1023,0.0303413321574529
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,1023,0.029909332593282063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,2047,0.051311999559402466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,4095,0.09532800316810608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,8191,0.17973866065343222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,4095,0.09431466460227966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,2047,0.052245333790779114
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,8191,0.18132267395655313
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,fp8,16383,0.35252265135447186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,1,0.00901333304742972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,7,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,3,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,3,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,1,128,1,float16,float16,16383,0.35071468353271484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,1,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,7,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,15,0.009808000177145004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,31,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,31,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,15,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,63,0.011690666278203329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,127,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,127,0.011781333635250727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,255,0.013568000247081121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,511,0.01903466631968816
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,255,0.01349866638580958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,63,0.011605333536863327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,1023,0.02985599885384242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,511,0.018911999960740406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,1023,0.030373332401116688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,2047,0.05208000044027964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,4095,0.09494933485984802
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,2047,0.0514933317899704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,4095,0.0952959954738617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,8191,0.1814240018526713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,fp8,16383,0.3526293436686198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,1,0.008943999807039896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,1,0.009141333401203156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,16383,0.35333867867787677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,2,128,1,float16,float16,8191,0.1804479956626892
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,3,0.009029333169261614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,7,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,15,0.009861333295702934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,3,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,7,0.009375999992092451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,31,0.011658667276302973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,63,0.011754666765530905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,63,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,127,0.011626667032639185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,255,0.013525333255529404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,31,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,127,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,255,0.013674666484196981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,511,0.018922666708628338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,1023,0.013760000467300415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,511,0.019343999524911244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,1023,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,2047,0.018805333723624546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,4095,0.020106667031844456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,4095,0.018191999445358913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,8191,0.03252800057331721
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,8191,0.0317493329445521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,1,0.008874666566650072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,16383,0.039834665755430855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,fp8,2047,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,4,128,1,float16,float16,16383,0.0379573330283165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,1,0.009093333035707474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,3,0.009194666519761086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,3,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,7,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,7,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,15,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,31,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,63,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,31,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,127,0.011829332758982977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,63,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,15,0.010122666756312052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,127,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,255,0.01333333303531011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,255,0.013829333086808523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,511,0.01322666679819425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,1023,0.010485333700974783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,511,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,1023,0.013647999614477158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,2047,0.012965332716703415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,4095,0.024160000185171764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,4095,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,8191,0.03488533447186152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,2047,0.02089600016673406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,16383,0.03435733417669932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,fp8,8191,0.02699733277161916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,1,0.0888426701227824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,128,8,128,1,float16,float16,16383,0.05115733544031779
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,1,0.09037866195042928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,3,0.09305066863695781
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,3,0.09225066502888997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,7,0.09674132863680522
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,15,0.12085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,15,0.11981333295504253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,31,0.12110933661460876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,7,0.0959999958674113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,31,0.11986666917800903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,127,0.14654933412869772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,63,0.12180800239245097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,63,0.12076800068219502
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,127,0.14569066961606345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,255,0.22686932484308878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,1023,0.710640033086141
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,255,0.22474133968353271
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,511,0.3869226773579915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,1023,0.7059360345204672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,2047,1.3623253504435222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,float16,511,0.3884693384170532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,1,0.08901866277058919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,1,128,1,float16,fp8,2047,1.3464214007059734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,1,0.09064533313115437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,7,0.09642666578292847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,3,0.09341333309809367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,3,0.09240000446637471
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,7,0.0976106623808543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,15,0.12079466382662456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,31,0.12190933028856914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,31,0.12071466445922852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,63,0.12249599893887837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,63,0.12204800049463908
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,15,0.12114666899045308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,127,0.14721600214640299
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,255,0.22808533906936646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,511,0.38920533657073975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,127,0.14713600277900696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,255,0.22791467110315958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,511,0.3889919916788737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,1023,0.716421365737915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,1,0.08913600444793701
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,1,0.09057600299517314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,2047,1.3544480005900066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,float16,2047,1.3648160298665364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,3,0.0925439993540446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,3,0.0934933324654897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,2,128,1,float16,fp8,1023,0.7070720195770264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,7,0.0969493289788564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,7,0.09731733798980713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,63,0.12283200025558472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,15,0.12154666582743327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,15,0.1209333340326945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,31,0.1218773325284322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,31,0.12114133437474568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,127,0.02012266715367635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,127,0.01759999990463257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,63,0.12174399693806966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,255,0.038917332887649536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,255,0.039018665750821434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,511,0.0740586668252945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,511,0.06002133091290792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,1023,0.07810666660467784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,float16,2047,0.10703999797503154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,1023,0.10761599739392598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,1,0.0888853371143341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,1,0.09064533313115437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,3,0.0925333301226298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,3,0.09368000427881877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,7,0.09752000371615092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,7,0.09672533472379048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,4,128,1,float16,fp8,2047,0.13675199945767721
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,15,0.12097066640853882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,15,0.12149866422017415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,31,0.12218133608500163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,31,0.12109866738319397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,63,0.12179199854532878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,255,0.052101333936055504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,63,0.12262399991353352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,127,0.027509334186712902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,255,0.03153600047032038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,511,0.06696000198523204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,127,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,1023,0.08276266853014629
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,1023,0.05978133281071981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,511,0.04141333450873693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,float16,2047,0.12257599830627441
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,128,8,128,1,float16,fp8,2047,0.08719999591509502
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,1,0.014671999961137772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,7,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,3,0.015237333873907724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,1,0.014954666296641031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,7,0.015295999745527903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,15,0.01640533283352852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,15,0.01640533283352852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,3,0.015103999525308609
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,31,0.019786667078733444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,31,0.01971199984351794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,63,0.019989332805077236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,63,0.020090666910012562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,127,0.01988799994190534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,511,0.03442133218050003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,127,0.020080000162124634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,255,0.023621333142121632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,1023,0.05542399982611338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,511,0.034458667039871216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,2047,0.09738666812578838
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,1023,0.056133334835370384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,255,0.023418667415777843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,2047,0.09858133395512898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,4095,0.18089600404103598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,8191,0.34831468264261883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,4095,0.1823413372039795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,8191,0.35042667388916016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,float16,16383,0.6839946905771891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,1,0.014933332800865173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,1,128,1,float16,fp8,16383,0.6854453086853027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,3,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,3,0.015125333021084467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,7,0.015130666395028433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,1,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,7,0.01543466622630755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,15,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,15,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,31,0.01970133309563001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,31,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,63,0.02000533292690913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,63,0.019760000209013622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,255,0.02364266663789749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,127,0.01998399943113327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,255,0.02348800003528595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,511,0.034634667138258614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,511,0.03450666616360346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,127,0.020010666300853092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,2047,0.09729599952697754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,1023,0.055386667450269066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,1023,0.05580799778302511
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,2047,0.0986293355623881
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,4095,0.18130133549372354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,4095,0.18266665935516357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,8191,0.3491520086924235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,float16,16383,0.7026826540629069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,1,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,16383,0.6852266788482666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,1,0.015072000523408255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,7,0.015034666905800501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,3,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,3,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,7,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,2,128,1,float16,fp8,8191,0.3499786853790283
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,15,0.016154666741689045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,15,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,31,0.019679999599854153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,31,0.019674666225910187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,63,0.019909333437681198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,63,0.019893333315849304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,127,0.019893333315849304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,255,0.02348800003528595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,255,0.023423999547958374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,511,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,511,0.012533333152532578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,127,0.019914666811625164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,1023,0.016165333489576977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,2047,0.025546667476495106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,1023,0.013983999689420065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,2047,0.024245334168275196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,4095,0.030031998952229817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,8191,0.04459733267625173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,4095,0.028021333118279774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,16383,0.0705866664648056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,fp8,8191,0.04419200122356415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,1,0.014912000546852747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,4,128,1,float16,float16,16383,0.06529599924882253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,3,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,1,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,3,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,7,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,7,0.015194666882356008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,15,0.016629333297411602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,31,0.01972266659140587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,31,0.019850666324297588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,15,0.016208000481128693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,63,0.019845332950353622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,63,0.019925333559513092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,127,0.020021333048741024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,127,0.02006400004029274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,255,0.012997332960367203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,255,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,511,0.013760000467300415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,1023,0.019914666811625164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,1023,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,2047,0.028890666862328846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,511,0.00973866693675518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,2047,0.020165332903464634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,4095,0.041120000183582306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,4095,0.023813332120577495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,8191,0.05914133290449778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,8191,0.037632000943024956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,float16,16383,0.08781866232554118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,128,8,128,1,float16,fp8,16383,0.06234133243560791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,1,0.1756640076637268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,1,0.17241599162419638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,7,0.18643200397491455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,3,0.17920533816019693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,3,0.18101332585016885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,15,0.23434666792551676
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,7,0.18774400154749551
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,15,0.23610132932662964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,31,0.23711466789245605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,63,0.23837866385777792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,63,0.23622934023539224
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,127,0.28702932596206665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,31,0.23440533876419067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,127,0.28520532449086505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,255,0.44706134001413983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,255,0.44290133317311603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,511,0.7663520177205404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,511,0.7635946273803711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,1,0.17590934038162231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,float16,1023,1.4098079999287922
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,1,128,1,float16,fp8,1023,1.3930932680765789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,1,0.17271999518076578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,3,0.17909866571426392
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,3,0.18164799610773721
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,7,0.18767466147740683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,15,0.23690666755040488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,7,0.18984532356262207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,31,0.23812800645828247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,15,0.23674132426579794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,63,0.23933867613474527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,63,0.2380160093307495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,127,0.2879413366317749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,31,0.2363413373629252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,127,0.2877546747525533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,255,0.44839998086293537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,255,0.4472320079803467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,511,0.7737279733022054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,float16,1023,1.412735939025879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,511,0.7653226852416992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,2,128,1,float16,fp8,1023,1.4033759435017903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,1,0.1763146718343099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,1,0.17262399196624756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,3,0.1788640022277832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,3,0.18141865730285645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,7,0.18779200315475464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,15,0.23669866720835367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,7,0.1900906761487325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,15,0.23709332942962646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,31,0.2382240096728007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,63,0.23965867360432944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,31,0.23658132553100586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,63,0.23825599749883017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,127,0.035216001172860466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,127,0.03245333333810171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,255,0.0670826683441798
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,255,0.07472533484299977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,511,0.08331199983755748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,511,0.11218133568763733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,float16,1023,0.10700800021489461
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,4,128,1,float16,fp8,1023,0.1399786671002706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,1,0.17613865931828818
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,1,0.17306667566299438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,3,0.17911465962727866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,7,0.18799465894699097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,3,0.1813919941584269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,7,0.19024533033370972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,15,0.2372693419456482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,31,0.23805334170659384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,15,0.2369119922320048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,31,0.23690134286880493
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,63,0.24025066693623862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,63,0.23835732539494833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,127,0.02722666660944621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,127,0.04875733455022176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,255,0.03700266778469086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,511,0.07299200197060902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,255,0.056133334835370384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,511,0.05525333185990652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,float16,1023,0.10809600353240967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,1,0.33951465288798016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,128,8,128,1,float16,fp8,1023,0.08260266482830048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,1,0.3452906608581543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,3,0.35317333539326984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,3,0.35711467266082764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,7,0.3670133352279663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,7,0.36977601051330566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,15,0.4635680119196574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,15,0.46673067410786945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,31,0.46860265731811523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,31,0.4633013407389323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,63,0.4712053140004476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,63,0.466810663541158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,127,0.5643680095672607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,255,0.889461358388265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,float16,127,0.5674773454666138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,1,128,1,float16,fp8,255,0.8791680335998535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,1,0.34011733531951904
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,1,0.34599467118581134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,3,0.35305599371592206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,7,0.3739146788914998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,3,0.3575466473897298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,7,0.3694560130437215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,31,0.47097599506378174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,15,0.46804265181223553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,31,0.46725332736968994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,15,0.46862932046254474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,63,0.4707253376642863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,63,0.4734453360239665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,127,0.571066657702128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,127,0.5697493155797323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,float16,255,0.8999626636505127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,2,128,1,float16,fp8,255,0.8899892965952555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,1,0.33980266253153485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,3,0.3526719808578491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,3,0.35703468322753906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,1,0.34628268082936603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,7,0.36956799030303955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,7,0.3744373321533203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,15,0.46910401185353595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,31,0.4707786639531453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,15,0.46847466627756756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,31,0.4679146607716878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,63,0.47669867674509686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,63,0.4707893530527751
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,127,0.06489600241184235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,127,0.06514666477839152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,float16,255,0.07382933298746745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,4,128,1,float16,fp8,255,0.10247466961542766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,1,0.34651732444763184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,1,0.3402453263600667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,3,0.35207998752593994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,7,0.3698986768722534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,7,0.37493332227071124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,3,0.35687466462453205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,31,0.47592000166575116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,15,0.4696799914042155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,15,0.46822933355967206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,31,0.4692320028940837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,63,0.4864159822463989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,63,0.4758666753768921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,127,0.08938666184743245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,float16,255,0.1018933355808258
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,127,0.06387199958165486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,128,8,128,1,float16,fp8,255,0.07738133271535237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,1,0.67413330078125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,1,0.6885333061218262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,3,0.703162670135498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,3,0.7105333010355631
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,7,0.7357493241628011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,7,0.7303626537322998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,15,0.9296533266703287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,15,0.9225333531697592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,31,0.9338133335113525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,31,0.9226293563842773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,63,0.9445226987202963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,63,0.9326293468475342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,float16,127,1.1442399819691975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,1,128,1,float16,fp8,127,1.1308213075002034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,1,0.6753173669179281
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,1,0.6883412996927897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,3,0.7033867041269938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,3,0.7115093072255453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,7,0.7371520201365153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,15,0.935802698135376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,7,0.7442506949106852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,15,0.9315946896870931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,31,0.9442613124847412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,31,0.9333653450012207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,63,0.9566826820373535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,63,0.9487199783325195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,float16,127,1.1549759705861409
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,2,128,1,float16,fp8,127,1.1532906691233318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,1,0.6759253342946371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,1,0.6894240379333496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,3,0.7031733194986979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,7,0.7428906758626302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,3,0.7116320133209229
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,7,0.7483092943827311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,15,0.9351360003153483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,15,0.940778652826945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,31,0.9521546363830566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,31,0.9428319931030273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,63,0.9598026275634766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,63,0.9592693646748861
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,float16,127,0.12154133121172588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,4,128,1,float16,fp8,127,0.17813867330551147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,1,0.6848106384277344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,1,0.6920373439788818
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,3,0.7110719680786133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,3,0.7148746649424235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,7,0.7586987018585205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,7,0.7583306630452474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,15,0.9483146667480469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,15,0.9443840185801188
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,31,0.9571519692738851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,31,0.9547093709309896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,63,0.9652693271636963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,1,0.01471466695268949
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,63,0.9663093090057373
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,float16,127,0.16699200868606567
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,1,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,128,8,128,1,float16,fp8,127,0.12270933389663696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,3,0.015024000157912573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,3,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,7,0.015850666910409927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,7,0.01590399940808614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,15,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,15,0.019280000279347103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,31,0.019578666736682255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,63,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,31,0.01934933289885521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,63,0.01941866676012675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,127,0.023306667804718018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,127,0.02309866746266683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,255,0.03400533397992452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,255,0.03396799912055334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,511,0.05586666862169901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,511,0.05580799778302511
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,1023,0.10030399759610494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,1023,0.09846400221188863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,2047,0.1839359998703003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,2047,0.18663465976715088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,4095,0.3550293445587158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,4095,0.3601333300272624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,8191,0.7094186941782633
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,8191,0.709274689356486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,float16,16383,1.6211412747701008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,1,128,1,float16,fp8,16383,1.6624479293823242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,1,0.014746667196353277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,1,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,3,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,3,0.015194666882356008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,7,0.015872000406185787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,7,0.015872000406185787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,15,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,15,0.01931200052301089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,127,0.023237332701683044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,31,0.019359999646743137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,31,0.019205333044131596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,63,0.019280000279347103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,63,0.019578666736682255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,127,0.0232640008131663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,255,0.03398400048414866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,255,0.03404266635576884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,511,0.0554720014333725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,511,0.056101332108179726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,1023,0.09827733039855957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,1023,0.10006933410962422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,2047,0.1853333314259847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,2047,0.18659200270970663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,4095,0.35971200466156006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,4095,0.3572213252385457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,8191,0.7183146476745605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,8191,0.7091733614603678
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,1,0.01488000030318896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,1,0.015103999525308609
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,float16,16383,1.7991894086201985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,2,128,1,float16,fp8,16383,1.7939573923746746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,3,0.015216000378131866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,3,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,7,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,7,0.015626666446526844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,15,0.019626667102177937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,15,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,31,0.019434666881958645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,31,0.019189332922299702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,63,0.019493332753578823
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,63,0.01940800001223882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,127,0.02325333406527837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,127,0.023344000180562336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,255,0.015386667102575302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,255,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,511,0.01573866605758667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,511,0.012448000411192576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,1023,0.023402666052182514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,1023,0.02502399931351344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,2047,0.03499733408292135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,2047,0.033370666205883026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,4095,0.051455999414126076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,4095,0.05295466880003611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,8191,0.0738453318675359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,8191,0.08107733229796092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,fp8,16383,0.13407466808954874
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,4,128,1,float16,float16,16383,0.10923199852307637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,1,0.014570667097965876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,1,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,3,0.015210667004187902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,3,0.01540800059835116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,7,0.015552000453074774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,7,0.015669333438078564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,15,0.01931200052301089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,15,0.01924266666173935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,31,0.01961600035429001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,31,0.019274666905403137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,63,0.01970133309563001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,255,0.013616000612576803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,255,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,63,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,127,0.010837333897749582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,511,0.020207999895016353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,127,0.008165333420038223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,1023,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,2047,0.0429066667954127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,511,0.010965333630641302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,1023,0.02861333390076955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,2047,0.027823999524116516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,4095,0.06734399994214375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,8191,0.08562666177749634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,4095,0.04208533465862274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,8191,0.060090666015942894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,float16,16383,0.12055466572443645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,128,8,128,1,float16,fp8,16383,0.08796266714731853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,1,1.3822879791259766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,1,1.413530667622884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,3,1.4399466514587402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,3,1.4585812886555989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,7,1.490565299987793
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,7,1.5043519337972004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,15,1.8665119806925456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,15,1.8567840258280437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,31,1.876736005147298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,31,1.8606079419453938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,float16,63,1.889786720275879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,1,128,1,float16,fp8,63,1.8735040028889973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,1,1.388912041982015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,1,1.419413407643636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,3,1.4411627451578777
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,3,1.4614559809366863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,7,1.5021920204162598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,7,1.519935925801595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,15,1.8767040570576985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,15,1.8759093284606934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,31,1.8825653394063313
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,31,1.8912266095479329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,float16,63,1.9063679377237956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,2,128,1,float16,fp8,63,1.9015146891276042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,1,1.3991573651631672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,1,1.430799961090088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,3,1.4434186617533367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,3,1.4655733108520508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,7,1.5099306106567383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,7,1.528714656829834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,15,1.8816053072611492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,15,1.882208029429118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,31,1.896608034769694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,float16,63,1.9127413431803386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,31,1.8911946614583333
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,4,128,1,float16,fp8,63,1.911402702331543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,1,1.4160480499267578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,1,1.4432907104492188
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,3,1.4512799580891926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,3,1.4747145970662434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,7,1.521077315012614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,15,1.8913386662801106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,7,1.5408533414204915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,15,1.8899307250976562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,31,1.9037334124247234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,63,1.9222346941630046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,fp8,63,1.9256319999694824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,128,8,128,1,float16,float16,31,1.9063092867533367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,float16,1,2.75767453511556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,fp8,1,2.818490664164225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,float16,3,2.8779519399007163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,fp8,3,2.9132054646809897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,float16,7,2.9742558797200522
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,float16,15,3.726053237915039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,fp8,7,3.0012693405151367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,fp8,15,3.706032117207845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,float16,31,3.7463626861572266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,float16,1,2.768890698750814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,1,128,1,float16,fp8,31,3.713360150655111
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,fp8,1,2.8331146240234375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,float16,3,2.8773120244344077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,fp8,3,2.9172213872273765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,float16,7,2.9970668156941733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,fp8,7,3.033232053120931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,float16,15,3.747354825337728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,fp8,15,3.7547518412272134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,float16,31,3.7745065689086914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,2,128,1,float16,fp8,31,3.759552001953125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,float16,1,2.791034698486328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,fp8,1,2.852970759073893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,float16,3,2.8784586588541665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,fp8,3,2.9251839319864907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,fp8,7,3.051637331644694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,float16,7,3.0134452184041343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,float16,15,3.7564853032430015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,fp8,15,3.7574612299601235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,fp8,31,3.774911880493164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,4,128,1,float16,float16,31,3.7866827646891275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,float16,1,2.8281758626302085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,fp8,1,2.8785813649495444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,float16,3,2.896357218424479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,fp8,3,2.942021369934082
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,float16,7,3.034063975016276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,fp8,7,3.07366943359375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,float16,15,3.772437413533529
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,fp8,15,3.7762505213419595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,float16,31,3.8041067123413086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,128,8,128,1,float16,fp8,31,3.8001972834269204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,1,0.025546667476495106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,1,0.02622399975856145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,3,0.026560001075267792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,3,0.02685333291689555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,7,0.02752533306678136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,7,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,15,0.03446933378775915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,15,0.034330666065216064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,31,0.03472000112136205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,31,0.03460799902677536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,63,0.034858666360378265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,63,0.03474666674931844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,127,0.041322665909926094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,127,0.041573333243529
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,255,0.06206400195757548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,255,0.06197333335876465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,511,0.10419733325640361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,511,0.10467200477917989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,1023,0.18755199511845908
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,1023,0.18837332725524902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,2047,0.3544693390528361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,2047,0.35518399874369305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,4095,0.687509298324585
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,4095,0.6881919701894125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,float16,8191,1.3741119702657063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,1,0.025674665967623394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,1,0.026202666262785595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,1,128,1,float16,fp8,8191,1.3603946367899578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,3,0.02649066597223282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,3,0.026816000541051228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,7,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,7,0.027615999182065327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,15,0.03437866767247518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,15,0.034559999903043113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,31,0.034976000587145485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,63,0.03510933369398117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,31,0.03480000048875809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,63,0.03482133398453394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,127,0.04128533353408178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,127,0.04118400067090988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,255,0.062165334820747375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,255,0.062319998939832054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,511,0.10461333394050598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,511,0.10413333773612976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,1023,0.1876479983329773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,2047,0.3548853397369385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,1023,0.1881386637687683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,2047,0.35524264971415204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,4095,0.6993813514709473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,float16,8191,1.3863892555236816
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,3,0.026848000784715016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,1,0.025818665822347004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,4095,0.6878933111826578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,3,0.02649066597223282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,2,128,1,float16,fp8,8191,1.3777440388997395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,1,0.02601066728432973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,15,0.03450666616360346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,7,0.02771199991305669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,7,0.027647999425729115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,15,0.034517332911491394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,31,0.034586665530999504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,31,0.03491200009981791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,63,0.03516799956560135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,127,0.012421333541472753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,63,0.03472000112136205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,127,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,255,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,255,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,511,0.023669332265853882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,511,0.02237333357334137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,1023,0.04153066625197729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,1023,0.034245334565639496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,2047,0.058975999553998314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,2047,0.05443733433882395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,1,0.025578667720158894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,4095,0.0846613347530365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,4095,0.10893866419792175
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,float16,8191,0.10706133643786113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,4,128,1,float16,fp8,8191,0.1360266705354055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,1,0.02622933437426885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,7,0.027461332579453785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,3,0.026608000199000042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,7,0.027808000644048054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,3,0.026858667532602947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,15,0.034815999368826546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,15,0.03460799902677536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,31,0.03475733349720637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,31,0.034789333740870156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,63,0.03498133271932602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,63,0.03486400097608566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,127,0.011045332998037338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,255,0.02160533269246419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,255,0.010709332923094431
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,511,0.028773332635561626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,127,0.00855466661353906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,511,0.018640000373125076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,1023,0.04658666749795278
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,1023,0.03267733256022135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,2047,0.06565333406130473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,1,0.025781333446502686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,2047,0.041936000188191734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,4095,0.08363200227419536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,4095,0.06006933252016703
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,fp8,8191,0.08720533053080241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,128,8,128,1,float16,float16,8191,0.11962133646011353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,1,0.02629866699377696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,3,0.027061333258946735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,3,0.026730666557947796
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,7,0.027679999669392902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,7,0.027658666173617046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,15,0.03444266567627589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,15,0.034671999514102936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,31,0.034847999612490334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,63,0.03498666733503342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,31,0.03459733227888743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,63,0.03481066723664602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,127,0.041738669077555336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,127,0.04133866727352142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,255,0.062394668658574425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,255,0.06233066817124685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,511,0.10401599605878194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,511,0.10479999581972758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,1023,0.1874506672223409
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,1023,0.18825066089630127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,2047,0.35471999645233154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,2047,0.3553546667098999
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,4095,0.6996800104777018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,4095,0.6885013580322266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,float16,8191,1.3893493016560872
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,1,128,1,float16,fp8,8191,1.3799626032511394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,1,0.025754667818546295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,1,0.026000000536441803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,3,0.026736001173655193
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,3,0.026816000541051228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,7,0.02749866743882497
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,7,0.027802666028340656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,15,0.03457066665093104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,15,0.03465066601832708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,31,0.03481066723664602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,31,0.03445333242416382
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,63,0.0351946676770846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,63,0.03489600121974945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,127,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,127,0.00980266680320104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,255,0.016010666886965435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,511,0.02348800003528595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,255,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,511,0.02236266682545344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,1023,0.03404266635576884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,1023,0.041152000427246094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,4095,0.08453866839408875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,4095,0.10538666447003682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,2047,0.05394133428732554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,2047,0.05899199843406677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,float16,8191,0.10808533430099487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,3,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,3,0.027029333015282948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,2,128,1,float16,fp8,8191,0.1351626714070638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,1,0.02589333305756251
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,1,0.02613866577545802
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,7,0.027855999767780304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,7,0.027589333554108936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,15,0.03449599941571554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,15,0.03453866640726725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,31,0.03475199888149897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,31,0.03453866640726725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,63,0.035258665680885315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,63,0.035088000198205314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,127,0.011301333705584208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,127,0.007861333588759104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,255,0.021546666820844013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,255,0.010608000059922537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,511,0.028762665887673695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,511,0.01842133328318596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,1023,0.032586666444937386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,1023,0.04650666813055674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,2047,0.06589866677920024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,2047,0.04178666571776072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,4095,0.08426666259765625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,4095,0.05972800155480703
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,float16,8191,0.12082667152086894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,4,128,1,float16,fp8,8191,0.08770666519800822
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,7,0.02775999903678894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,1,0.026101333399613697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,3,0.026911998788515728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,15,0.03458133339881897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,31,0.03485333422819773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,63,0.03494933247566223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,127,0.007381333038210869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,1,0.025754667818546295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,3,0.026880001028378803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,15,0.034586665530999504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,7,0.02757333219051361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,31,0.03489600121974945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,511,0.04663999875386556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,255,0.030378667016824085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,63,0.03528533379236857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,1023,0.06135466694831848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,1023,0.0276853342851003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,127,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,2047,0.08335999647776286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,2047,0.04113066693147024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,4095,0.1188159982363383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,255,0.015040000279744467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,float16,8191,0.1900426745414734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,1,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,8191,0.1002346674601237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,511,0.021727999051411945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,1,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,7,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,7,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,64,8,128,1,float16,fp8,4095,0.0612960010766983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,15,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,31,0.01020800011853377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,15,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,63,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,31,0.010319999729593595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,63,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,255,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,255,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,127,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,511,0.016741332908471424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,511,0.016864000509182613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,1023,0.029685333371162415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,1023,0.029626667499542236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,2047,0.03728000074625015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,2047,0.03746666759252548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,4095,0.051413332422574363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,8191,0.08108266691366832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,4095,0.05198933184146881
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,16383,0.13893866539001465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,8191,0.08089066545168559
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,16383,0.1379093329111735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,1,0.009674666449427605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,float16,32767,0.25514666239420575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,1,128,1,float16,fp8,32767,0.2531733314196269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,1,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,3,0.009610666582981745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,7,0.009450666606426239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,15,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,15,0.009712000067035357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,31,0.01028266673286756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,63,0.012053333222866058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,31,0.010202666744589806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,63,0.012191999703645706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,127,0.012149333953857422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,127,0.012223999947309494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,255,0.012416000167528788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,255,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,511,0.016613333175579708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,511,0.016762666404247284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,1023,0.029520000020662945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,1023,0.02958400050799052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,2047,0.037418665985266365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,2047,0.03736533224582672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,4095,0.018383999665578205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,4095,0.018058666338523228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,8191,0.02269333352645238
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,8191,0.021909333765506744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,16383,0.026373334228992462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,16383,0.025263999899228413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,float16,32767,0.030597334106763203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,2,128,1,float16,fp8,32767,0.03403199960788091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,1,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,7,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,3,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,7,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,3,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,15,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,31,0.010314666976531347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,31,0.010266666611035665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,63,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,63,0.011994666109482447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,127,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,255,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,127,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,511,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,255,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,1023,0.029696000119050343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,511,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,1023,0.029520000020662945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,2047,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,2047,0.01146666705608368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,4095,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,4095,0.013066666821638743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,8191,0.01603200038274129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,16383,0.02145066608985265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,8191,0.01934933289885521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,16383,0.019152000546455383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,fp8,32767,0.028746667007605236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,4,128,1,float16,float16,32767,0.035002666215101876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,1,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,3,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,3,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,7,0.009583999713261923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,15,0.009519999846816063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,31,0.010213333492477735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,255,0.012293333808581034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,63,0.012058666596810022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,127,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,63,0.012042666474978128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,127,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,15,0.010101333260536194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,511,0.016783999900023144
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,255,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,511,0.016842667013406754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,1023,0.013658666362365087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,1023,0.00860799973209699
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,2047,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,2047,0.015290666371583939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,4095,0.016069332758585613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,4095,0.010879999647537867
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,8191,0.014981333166360855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,16383,0.030165334542592365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,16383,0.018144000321626663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,8191,0.025349333882331848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,float16,32767,0.04829333225886027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,64,8,128,1,float16,fp8,32767,0.0272533322374026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,1,0.009685333197315535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,1,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,3,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,3,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,7,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,15,0.0099093330403169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,7,0.009525333220760027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,31,0.010543999572594961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,15,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,31,0.010474666953086853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,63,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,63,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,127,0.012426666915416718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,127,0.012373333175977072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,255,0.012549333274364471
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,255,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,511,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,511,0.028186666468779247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,1023,0.03271999955177307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,1023,0.03260799994071325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,2047,0.04410133262475332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,2047,0.043807998299598694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,4095,0.06593066453933716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,8191,0.10890666643778484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,4095,0.06565866867701213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,8191,0.1090186635653178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,16383,0.19520533084869385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,1,0.009701333319147428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,16383,0.19614932934443155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,float16,32767,0.3706933259963989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,1,128,1,float16,fp8,32767,0.36814932028452557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,1,0.009925333162148794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,3,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,7,0.00979200005531311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,3,0.009829333052039146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,7,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,15,0.009973333527644476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,15,0.009632000078757605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,31,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,31,0.010474666953086853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,63,0.012234666695197424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,255,0.012538666526476542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,63,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,127,0.012448000411192576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,255,0.012266666938861212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,511,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,511,0.02809600035349528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,1023,0.032613334556420646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,1023,0.0327360009153684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,2047,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,2047,0.018245333184798557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,4095,0.019285333653291065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,8191,0.023893333971500397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,4095,0.01798933371901512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,8191,0.02187199890613556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,32767,0.04489066700140635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,16383,0.026591998835404713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,float16,16383,0.026074667771657307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,2,128,1,float16,fp8,32767,0.049322664737701416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,1,0.009578666960199675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,3,0.009583999713261923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,3,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,1,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,7,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,15,0.010090666512648264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,15,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,31,0.010597333312034607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,31,0.01062400018175443
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,63,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,127,0.012335999558369318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,255,0.01258133351802826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,255,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,511,0.028250666956106823
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,511,0.02812266598145167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,2047,0.015813333292802174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,1023,0.010378666842977205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,1023,0.013418667018413544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,4095,0.017093333105246227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,4095,0.013359999905029932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,8191,0.02626666675011317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,2047,0.011658667276302973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,8191,0.018602666755517323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,16383,0.031328000128269196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,16383,0.02293333411216736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,float16,32767,0.04957866668701172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,1,0.009813333551088968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,4,128,1,float16,fp8,32767,0.04146133363246918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,1,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,7,0.009663999701539675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,3,0.009808000177145004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,3,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,7,0.009775999933481216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,15,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,15,0.010005333150426546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,31,0.010357333347201347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,31,0.010431999961535135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,127,0.012346666306257248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,63,0.012053333222866058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,255,0.012186666329701742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,255,0.012522666404644648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,511,0.01303999995191892
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,511,0.008005333443482717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,1023,0.013338666409254074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,1023,0.008656000097592672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,2047,0.020432000358899433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,2047,0.010832000523805618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,4095,0.023706667125225067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,4095,0.012634667257467905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,8191,0.035162667433420815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,8191,0.01681600014368693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,16383,0.05040533343950907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,16383,0.02899733434120814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,float16,32767,0.07760533193747203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,64,8,128,1,float16,fp8,32767,0.041264000038305916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,1,0.04700266818205515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,1,0.047877331574757896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,3,0.0487413356701533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,3,0.04987200101216634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,7,0.05109866460164388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,15,0.06366933385531108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,31,0.06387199958165486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,15,0.06332266827424367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,7,0.05087999999523163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,31,0.06345066428184509
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,63,0.06457066535949707
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,127,0.07720533510049184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,127,0.07719466586907704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,63,0.06429333488146464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,255,0.11851200461387634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,255,0.11814399560292561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,511,0.20103466510772705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,511,0.20091732343037924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,1023,0.3654346863428752
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,1023,0.36449599266052246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,2047,0.69923202196757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,2047,0.6919039885203043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,fp8,4095,1.3558346430460613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,1,0.04722133278846741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,1,128,1,float16,float16,4095,1.362186590830485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,3,0.04930666585763296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,1,0.04814933240413666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,3,0.04974400003751119
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,15,0.06316266457239787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,7,0.050986667474110924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,15,0.06384533147017162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,7,0.051221330960591636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,31,0.0642080008983612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,31,0.06359999875227611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,63,0.06464000046253204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,63,0.06422933439413707
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,127,0.012831999609867731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,255,0.026015999416510265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,127,0.010101333260536194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,511,0.03458133339881897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,255,0.021397332350413006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,511,0.040048000713189445
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,1023,0.05787733197212219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,1023,0.07321600119272868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,2047,0.08328533172607422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,2047,0.10743467013041179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,fp8,4095,0.13547733426094055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,2,128,1,float16,float16,4095,0.10678399602572124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,1,0.04741866886615753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,1,0.04795200129350027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,3,0.04906666775544485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,3,0.049839998284975685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,7,0.05097599824269613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,7,0.051258668303489685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,15,0.06371200084686279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,31,0.06406933565934499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,15,0.06357333560784657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,31,0.0637546678384145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,63,0.06454933186372121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,63,0.06426133215427399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,127,0.016197333733240765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,127,0.008736000085870424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,255,0.03108799954255422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,255,0.017765333255132038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,511,0.04756799836953481
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,511,0.03198933353026708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,1023,0.06225599845250448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,2047,0.08433600266774495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,1023,0.04173333446184794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,float16,4095,0.12105066577593486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,2047,0.05893866717815399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,4,128,1,float16,fp8,4095,0.08705600102742513
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,1,0.047151997685432434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,1,0.04809600114822388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,3,0.0490880012512207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,7,0.051269332567850746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,3,0.049642667174339294
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,7,0.051072001457214355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,15,0.06361599763234456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,15,0.06377600133419037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,31,0.06390400230884552
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,31,0.06436266501744588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,63,0.06412800153096516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,127,0.02588266630967458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,63,0.06471466521422069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,127,0.012144000579913458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,255,0.021840001145998638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,255,0.0498879998922348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,1023,0.0806879997253418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,1023,0.04127999891837438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,511,0.06483200192451477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,511,0.0278613343834877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,2047,0.1204800009727478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,2047,0.062309334675470986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,float16,4095,0.1913493275642395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,1,0.009061333412925402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,1,0.009136000027259191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,3,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,64,8,128,1,float16,fp8,4095,0.09990933537483215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,3,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,7,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,7,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,15,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,31,0.011525332927703857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,31,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,63,0.011792000383138657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,63,0.011685332904259363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,127,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,127,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,255,0.013712000101804733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,255,0.013461332768201828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,511,0.0191040001809597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,511,0.019141333798567455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,1023,0.029994666576385498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,2047,0.05148266752560934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,1023,0.03038399914900462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,4095,0.09530666470527649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,2047,0.05209066470464071
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,4095,0.09423466523488362
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,8191,0.18000000715255737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,8191,0.18134399255116782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,16383,0.35212798913319904
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,16383,0.35099732875823975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,float16,32767,0.8691573143005371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,1,128,1,float16,fp8,32767,0.8141653537750244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,1,0.009072000160813332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,1,0.008992000172535578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,3,0.009098666409651438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,3,0.009375999992092451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,15,0.010037333394090334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,31,0.011557333171367645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,7,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,15,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,31,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,7,0.009322666873534521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,63,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,63,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,127,0.011663999408483505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,127,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,255,0.013466666142145792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,1023,0.015919999529918034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,255,0.013658666362365087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,1023,0.013834666460752487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,511,0.0191040001809597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,511,0.018986667195955913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,2047,0.019968000551064808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,2047,0.015557333827018738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,4095,0.018325333793958027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,4095,0.02146666745344798
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,8191,0.032613334556420646
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,16383,0.039834665755430855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,8191,0.03293866664171219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,16383,0.03939199944337209
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,float16,32767,0.0653599997361501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,2,128,1,float16,fp8,32767,0.06934933364391327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,1,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,1,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,3,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,15,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,3,0.00903466654320558
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,7,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,7,0.009205333267649015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,15,0.009941333283980688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,31,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,63,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,63,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,127,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,31,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,127,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,255,0.013557333499193192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,255,0.013760000467300415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,511,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,511,0.01292266696691513
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,1023,0.013744000345468521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,1023,0.010602666685978571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,2047,0.022015998760859173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,2047,0.013199999928474426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,4095,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,4095,0.015205333630243937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,8191,0.03666666646798452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,8191,0.026928000152111053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,16383,0.03456533451875051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,16383,0.0544106662273407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,1,0.009050666665037474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,3,0.009050666665037474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,float16,32767,0.08116266628106435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,1,0.009082666908701261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,4,128,1,float16,fp8,32767,0.06042666733264923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,3,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,7,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,7,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,15,0.010133333504199982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,31,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,15,0.010005333150426546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,63,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,63,0.011754666765530905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,31,0.011616000284751257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,127,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,127,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,255,0.012485332787036896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,255,0.008058666562040647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,511,0.013157332936922709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,1023,0.01952533299724261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,511,0.008165333420038223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,1023,0.009626666704813639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,2047,0.02863466739654541
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,4095,0.0401706670721372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,2047,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,8191,0.05829866727193197
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,4095,0.020154666155576706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,8191,0.025706666211287182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,16383,0.08602133393287659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,float16,32767,0.12273599704106648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,16383,0.045141334335009255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,64,8,128,1,float16,fp8,32767,0.06622933348019917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,1,0.08993599812189738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,3,0.0928000013033549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,1,0.09131733576456706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,3,0.09407466650009155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,7,0.09634133179982503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,7,0.09712533156077068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,15,0.12095999717712402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,15,0.12124266227086385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,31,0.12080533305803935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,31,0.12226667006810506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,63,0.12293333808581035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,63,0.12256000439325969
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,127,0.14748266339302063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,255,0.22845333814620972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,127,0.14756266276041666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,255,0.22823999325434366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,511,0.3895893494288127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,511,0.3893973429997762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,1023,0.7169386545817057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,1023,0.7077386379241943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,fp8,2047,1.3544960021972656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,1,128,1,float16,float16,2047,1.3633707364400227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,1,0.08958933750788371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,3,0.09303466478983562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,7,0.09635200103123982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,1,0.09143466750780742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,3,0.09388800462086995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,7,0.09739200274149577
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,15,0.12149332960446675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,31,0.12206400434176128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,15,0.12121599912643433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,127,0.020080000162124634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,63,0.12314132849375407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,31,0.12115200360616048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,63,0.12301333745320638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,127,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,255,0.038890667259693146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,255,0.03866666555404663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,511,0.07534933090209961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,511,0.05991466840108236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,1023,0.10763733585675557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,2047,0.10771733522415161
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,float16,1023,0.07797333101431529
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,1,0.08970133463541667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,2,128,1,float16,fp8,2047,0.13775466879208884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,3,0.09291733304659526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,1,0.09129066268603007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,3,0.09412800272305806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,7,0.09666132926940918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,7,0.09725866715113322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,15,0.12099200487136841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,15,0.12155200044314067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,31,0.12098667025566101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,31,0.12223999698956807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,63,0.12294933199882507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,63,0.12269866466522217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,127,0.027637332677841187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,127,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,255,0.05194133520126343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,511,0.06663466493288676
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,255,0.03152533372243246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,511,0.04229333500067393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,1023,0.08302400012811025
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,1023,0.05989866455396017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,float16,2047,0.12299733360608418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,4,128,1,float16,fp8,2047,0.08819733063379924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,1,0.09150933225949605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,1,0.08982933561007182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,3,0.09301867087682088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,3,0.09442133704821269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,7,0.09659733374913533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,7,0.0976639986038208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,31,0.12226667006810506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,15,0.12173333764076233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,15,0.12124266227086385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,31,0.12133333086967468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,63,0.12315733234087627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,63,0.12299733360608418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,127,0.04620266457398733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,255,0.05354666709899902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,127,0.01757866640885671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,255,0.0249439999461174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,511,0.03757333258787791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,511,0.07054399947325389
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,1023,0.10444266597429912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,1023,0.05710400144259135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,fp8,2047,0.09667733311653137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,64,8,128,1,float16,float16,2047,0.1767253279685974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,1,0.1739573280016581
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,3,0.17991999785105386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,3,0.1822133262952169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,1,0.17754133542378744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,7,0.18800000349680582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,15,0.2372693419456482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,7,0.18976000944773355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,15,0.23677333196004233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,31,0.23854933182398477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,31,0.23662400245666504
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,63,0.24109333753585815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,127,0.28810133536656696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,63,0.23973333835601807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,127,0.28904000918070477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,255,0.44887999693552655
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,255,0.4479999939600627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,float16,511,0.7740000089009603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,1,128,1,float16,fp8,511,0.7660000324249268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,1,0.17402666807174683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,1,0.1774239937464396
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,3,0.1800266702969869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,3,0.1823093295097351
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,7,0.18825600544611612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,7,0.1897760033607483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,15,0.237445334593455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,15,0.23703465859095255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,31,0.23896533250808716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,31,0.23686933517456055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,63,0.23941334088643393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,63,0.2405973275502523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,127,0.03489600121974945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,127,0.032501332461833954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,255,0.07696533203125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,255,0.06669333577156067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,1,0.17407466967900595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,fp8,511,0.1104746659596761
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,2,128,1,float16,float16,511,0.08303466439247131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,3,0.1798400084177653
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,1,0.17747733990351358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,3,0.18219733238220215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,7,0.1880693236986796
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,7,0.19010667006174722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,15,0.23732266823450723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,15,0.23706666628519693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,31,0.23898667097091675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,31,0.2371413310368856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,127,0.0487306664387385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,63,0.23966399828592935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,63,0.2412266731262207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,127,0.02640533447265625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,255,0.05635199944178263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,255,0.03736533224582672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,float16,511,0.07417599856853485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,4,128,1,float16,fp8,511,0.0552106648683548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,1,0.17415465911229452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,1,0.17748800913492838
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,7,0.1881493330001831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,3,0.1800640026728312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,3,0.18262932697931925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,7,0.19065600633621216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,15,0.23787200450897217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,15,0.23757867018381754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,31,0.2394719918568929
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,63,0.24973867336908975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,63,0.2412053346633911
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,127,0.03320533285538355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,31,0.2371786634127299
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,127,0.08530132969220479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,255,0.09657599528630574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,255,0.04747733473777771
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,float16,511,0.12869866689046225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,64,8,128,1,float16,fp8,511,0.06653866668542226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,1,0.34350399176279706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,1,0.3498026529947917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,3,0.35604798793792725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,3,0.36025599638621014
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,7,0.3702346483866374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,7,0.3739306529362996
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,15,0.46906133492787677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,15,0.468394676844279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,31,0.47197333971659344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,31,0.46825067202250165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,63,0.473685344060262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,63,0.47518932819366455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,127,0.573632001876831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,127,0.5720853408177694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,float16,255,0.9004053274790446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,1,128,1,float16,fp8,255,0.8908747037251791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,1,0.34994133313496906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,1,0.3437439997990926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,3,0.3545653422673543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,3,0.3599626620610555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,7,0.3709760109583537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,7,0.3747573296229045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,15,0.4692106644312541
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,15,0.4686719973882039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,31,0.47226667404174805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,31,0.4686453342437744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,63,0.478218674659729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,63,0.47377065817515057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,127,0.06589866677920024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,255,0.07292266686757405
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,float16,127,0.06491200129191081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,2,128,1,float16,fp8,255,0.10293866197268169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,1,0.3436586856842041
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,1,0.35018134117126465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,3,0.3545120159784953
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,3,0.35971732934316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,7,0.37096532185872394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,7,0.37482666969299316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,15,0.4689653317133586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,15,0.4698559840520223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,31,0.4763733148574829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,31,0.46938133239746094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,63,0.4885439872741699
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,63,0.4783039887746175
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,127,0.08986133337020874
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,127,0.06385066608587901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,float16,255,0.10213866829872131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,1,0.34346667925516766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,4,128,1,float16,fp8,255,0.07794666786988576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,1,0.3502720197041829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,3,0.3550399939219157
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,3,0.36001598834991455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,7,0.37231465180714923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,15,0.47513067722320557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,7,0.37566932042439777
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,15,0.4704853296279907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,31,0.48691733678181964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,31,0.47729599475860596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,63,0.4918453296025594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,63,0.4920320113499959
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,127,0.06586666901906331
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,127,0.16089066863059998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,1,0.014485333114862442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,float16,255,0.18219733238220215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,64,8,128,1,float16,fp8,255,0.08469866712888081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,1,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,3,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,3,0.01533866673707962
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,7,0.014906667172908783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,7,0.01544533297419548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,15,0.016389333953460056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,15,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,31,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,31,0.019797333826621372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,63,0.01995733380317688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,63,0.01979200045267741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,127,0.020026666422684986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,255,0.023557332654794056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,127,0.02019199977318446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,511,0.034485332667827606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,255,0.023552000522613525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,511,0.034661332766215004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,1023,0.055200000603993736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,1023,0.05598400036493937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,2047,0.09723732868830363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,2047,0.09822400410970052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,4095,0.18253332376480103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,8191,0.3487786849339803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,4095,0.18100800116856894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,8191,0.35023999214172363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,16383,0.6994346777598063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,16383,0.6850720246632894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,1,0.014885333677132925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,float16,32767,1.7725866635640461
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,1,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,1,128,1,float16,fp8,32767,1.7725599606831868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,3,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,3,0.015418666104475657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,7,0.015205333630243937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,7,0.015322666615247726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,15,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,15,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,31,0.019808000574509304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,31,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,63,0.02000533292690913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,63,0.019952000429232914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,255,0.023530667026837666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,255,0.023520000278949738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,127,0.020074666788180668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,127,0.020021333048741024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,511,0.01259200026591619
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,511,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,1023,0.01563199982047081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,1023,0.013978666315476099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,2047,0.02590399980545044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,2047,0.024090667565663654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,4095,0.02998399982849757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,4095,0.028149334092934925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,8191,0.04529066880544027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,8191,0.04410133262475332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,16383,0.06603200236956279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,16383,0.07554133236408234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,float16,32767,0.10321600238482158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,3,0.015050667027632395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,2,128,1,float16,fp8,32767,0.1267039974530538
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,1,0.015040000279744467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,1,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,3,0.015333333363135656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,7,0.015178666760524115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,15,0.01661866654952367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,15,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,7,0.015413332730531693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,31,0.019871999820073444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,31,0.01972266659140587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,63,0.019914666811625164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,63,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,127,0.020303999384244282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,127,0.019904000063737232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,511,0.013290667285521826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,255,0.012768000364303589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,255,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,511,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,1023,0.011663999408483505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,1023,0.019365333020687103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,2047,0.028789333999156952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,2047,0.019845332950353622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,4095,0.04050666590531667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,4095,0.023818666736284893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,8191,0.059061333537101746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,8191,0.03696000079313914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,16383,0.08776533603668213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,16383,0.06243733565012614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,1,0.014906667172908783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,float16,32767,0.12434132893880208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,4,128,1,float16,fp8,32767,0.097653329372406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,1,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,3,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,3,0.015360000232855478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,7,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,15,0.016293333222468693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,7,0.01534933348496755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,15,0.01685333376129468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,31,0.019760000209013622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,31,0.019962667177120846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,63,0.02014933278163274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,63,0.019962667177120846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,127,0.010399999717871347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,127,0.007258666679263115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,255,0.01349866638580958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,511,0.01930133377512296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,255,0.008778666456540426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,511,0.009455999980370203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,1023,0.02812266598145167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,1023,0.015541333705186844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,2047,0.018800000349680584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,2047,0.04312000175317129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,4095,0.028624000648657482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,4095,0.06638933221499126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,8191,0.042506664991378784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,8191,0.08410666386286418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,16383,0.12000000476837158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,16383,0.060736000537872314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,float16,32767,0.19007466236750284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,64,8,128,1,float16,fp8,32767,0.10188800096511841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,1,0.6825973192850748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,1,0.6972373326619467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,3,0.7127093474070231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,3,0.7191733519236246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,7,0.7385439872741699
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,7,0.7444159984588623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,15,0.9360213279724121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,15,0.9333600203196207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,31,0.9474666913350424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,31,0.9352693557739258
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,63,0.9614400068918864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,63,0.9535360336303711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,1,0.6846933364868164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,float16,127,1.158746639887492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,1,128,1,float16,fp8,127,1.157274643580119
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,1,0.6982133388519287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,3,0.7119999726613363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,3,0.7192266782124838
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,7,0.745189348856608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,7,0.7487786610921224
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,15,0.942074696222941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,15,0.9372586409250895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,31,0.9542826811472574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,31,0.9458080132802328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,63,0.963813304901123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,63,0.9638720353444418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,float16,127,0.12171199917793274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,2,128,1,float16,fp8,127,0.17815999190012613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,1,0.6877493063608805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,1,0.6993707021077474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,3,0.7171359856923422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,3,0.7201866308848063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,7,0.7597653071085612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,7,0.759663979212443
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,15,0.9462026755015055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,15,0.9487946828206381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,31,0.9595466454823812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,31,0.9576480388641357
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,63,0.9691147009531657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,63,0.9711413383483887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,float16,127,0.16778665781021118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,4,128,1,float16,fp8,127,0.12329600254694621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,1,0.6996959845225016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,1,0.7082986831665039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,3,0.7320373058319092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,7,0.778335968653361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,3,0.7354400157928467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,7,0.7698559761047363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,15,0.9541172981262207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,15,0.9566400051116943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,31,0.9657546679178873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,63,0.9759946664174398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,31,0.9647573630015055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,63,0.9778347015380859
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,float16,127,0.310698668162028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,64,8,128,1,float16,fp8,127,0.12325867017110188
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,1,1.3985652923583984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,1,1.4274346033732097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,3,1.453781286875407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,3,1.470688025156657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,7,1.506480058034261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,7,1.5238666534423828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,15,1.8801280657450359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,15,1.8802879651387532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,31,1.897509256998698
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,float16,63,1.9156640370686848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,31,1.8880160649617512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,1,1.4046026865641277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,1,128,1,float16,fp8,63,1.911194642384847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,3,1.4731146494547527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,1,1.4367039998372395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,3,1.451797326405843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,7,1.5328853925069172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,7,1.5134666760762532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,15,1.8843679428100586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,15,1.8862773577372234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,31,1.90228271484375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,31,1.8960053126017253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,float16,63,1.9215893745422363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,2,128,1,float16,fp8,63,1.9217119216918945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,1,1.4477119445800781
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,1,1.4212533632914226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,3,1.4816320737202961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,3,1.4580426216125488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,7,1.5233813921610515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,7,1.5436959266662598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,15,1.8921279907226562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,15,1.8954453468322754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,31,1.9086880683898926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,31,1.911253293355306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,float16,63,1.9312373797098796
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,4,128,1,float16,fp8,63,1.9352800051371257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,1,1.4275093078613281
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,1,1.4576640129089355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,3,1.4665600458780925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,3,1.4909440676371257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,7,1.5332214037577312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,7,1.5553174018859863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,15,1.901093324025472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,15,1.907365322113037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,31,1.924239953358968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,1,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,1,0.014949332922697067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,3,0.015210667004187902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,31,1.9221760431925456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,7,0.015829333414634068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,3,0.015274666249752045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,float16,63,1.944976011912028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,64,8,128,1,float16,fp8,63,1.9483680725097656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,7,0.015722667177518208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,15,0.019402666638294857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,15,0.019109333554903667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,31,0.019258666783571243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,31,0.0194560003777345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,63,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,63,0.019472000499566395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,127,0.023210667073726654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,255,0.034272000193595886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,255,0.034234667817751564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,127,0.02327999969323476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,511,0.05551999807357788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,511,0.05607999861240387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,1023,0.09828266501426697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,1023,0.10004799564679463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,2047,0.18371200561523438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,2047,0.18673600753148398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,4095,0.3549813429514567
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,8191,0.724613348642985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,4095,0.3598826726277669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,8191,0.7091306845347086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,float16,16383,1.7744533220926921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,1,0.014655999839305878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,1,0.014853333433469137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,1,128,1,float16,fp8,16383,1.8184800148010254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,3,0.01505600040157636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,3,0.01525866612792015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,7,0.015840000162522
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,7,0.015754666179418564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,15,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,15,0.019098666807015736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,31,0.019434666881958645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,63,0.019509332875410717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,31,0.019248000035683315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,127,0.02332799881696701
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,63,0.01956266661485036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,127,0.023189333577950794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,255,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,255,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,511,0.015386667102575302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,511,0.012517333030700684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,1023,0.023221333821614582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,1023,0.025226667523384094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,2047,0.035018667578697205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,4095,0.05120533208052317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,2047,0.033359999457995095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,4095,0.05273066461086273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,8191,0.07849599917729695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,8191,0.07415999968846639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,float16,16383,0.11025066177050273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,1,0.014741333822409311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,2,128,1,float16,fp8,16383,0.13371200362841287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,1,0.014848000059525171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,3,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,7,0.015829333414634068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,3,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,7,0.01570133368174235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,15,0.019402666638294857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,15,0.01933866615096728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,31,0.019567999988794327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,31,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,63,0.01939733326435089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,63,0.01937066639463107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,127,0.010608000059922537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,127,0.008309333274761835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,255,0.009466666728258133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,255,0.013738666971524557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,511,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,511,0.010863999525705973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,1023,0.028192001084486645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,1023,0.01956266661485036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,2047,0.04369066655635834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,2047,0.02740799884001414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,4095,0.06824000179767609
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,8191,0.060005332032839455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,4095,0.04234666625658671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,8191,0.08553066849708557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,float16,16383,0.12178132931391399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,1,0.014991999914248785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,1,0.014757333944241205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,4,128,1,float16,fp8,16383,0.08775466680526733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,3,0.015247999380032221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,7,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,7,0.01602666700879733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,3,0.015173333386580149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,15,0.019333332777023315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,15,0.019402666638294857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,31,0.01947733387351036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,31,0.01926933353145917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,63,0.019679999599854153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,127,0.007247999931375186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,63,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,127,0.010714666297038397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,255,0.020938667158285778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,255,0.008879999940594038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,511,0.028058665494124096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,511,0.01498666654030482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,1023,0.04558933277924856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,1023,0.021701333423455555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,2047,0.06437866886456807
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,2047,0.028170667588710785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,4095,0.082997332016627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,4095,0.04142933338880539
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,8191,0.11906666556994121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,8191,0.06305066744486491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,float16,16383,0.18847467501958212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,64,8,128,1,float16,fp8,16383,0.1011199951171875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,1,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,3,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,3,0.015226667126019796
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,1,0.014783999572197596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,7,0.01579733317097028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,7,0.01569066693385442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,31,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,15,0.019509332875410717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,31,0.019498666127522785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,15,0.01941866676012675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,63,0.01951466624935468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,63,0.019333332777023315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,127,0.023205332458019257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,127,0.023242667317390442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,255,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,511,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,255,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,1023,0.022970666488011677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,511,0.012666666259368261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,1023,0.025055999557177227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,2047,0.0347626656293869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,4095,0.051039998730023704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,2047,0.03324799984693527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,4095,0.05292266607284546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,8191,0.0809440016746521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,8191,0.07435733576615651
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,float16,16383,0.11115733782450359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,1,128,1,float16,fp8,16383,0.1342026690642039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,1,0.014757333944241205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,1,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,3,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,3,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,7,0.015728000551462173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,7,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,15,0.01961600035429001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,15,0.01916266605257988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,31,0.01947733387351036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,31,0.019248000035683315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,63,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,63,0.019440000255902607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,127,0.010559999694426855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,127,0.00749333327015241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,255,0.013616000612576803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,511,0.010794666906197866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,255,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,1023,0.027962667246659596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,511,0.019626667102177937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,1023,0.019482667247454327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,2047,0.04394666850566864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,2047,0.027664000789324444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,4095,0.068122665087382
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,4095,0.04267199834187826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,8191,0.06021333237489065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,8191,0.08540800213813782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,float16,16383,0.12154133121172588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,1,0.014592000593741735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,1,0.015002666662136713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,2,128,1,float16,fp8,16383,0.08833600083986919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,3,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,3,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,7,0.015610666324694952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,7,0.015829333414634068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,15,0.01960533360640208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,15,0.01926933353145917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,31,0.019658666104078293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,31,0.019253333409627277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,63,0.019610666980346043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,63,0.01940800001223882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,127,0.010751999914646149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,127,0.007071999832987785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,255,0.021040000021457672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,255,0.008837333569924036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,511,0.02794133375088374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,511,0.014815999815861383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,1023,0.0454720010360082
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,1023,0.021738665799299877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,2047,0.0651146670182546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,2047,0.027471999327341717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,4095,0.08343467116355896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,4095,0.041477332512537636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,8191,0.11924800276756287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,8191,0.06330133477846782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,fp8,16383,0.10103999574979146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,4,128,1,float16,float16,16383,0.19115734100341797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,3,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,1,0.014837333311637243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,15,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,7,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,31,0.019386666516462963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,63,0.0195573332409064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,127,0.006762666627764702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,1,0.014661333213249842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,127,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,3,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,7,0.015861333658297855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,15,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,31,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,511,0.04619733492533366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,63,0.019637333850065868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,255,0.030192000170548756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,1023,0.061050668358802795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,255,0.01313599944114685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,2047,0.08274133503437042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,2047,0.03818133225043615
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,4095,0.059290667374928795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,4095,0.11845866839090984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,8191,0.19002666076024374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,511,0.01985599969824155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,8191,0.09771733482678731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,float16,16383,0.3296266595522563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,1023,0.025685332715511322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,1,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,32,8,128,1,float16,fp8,16383,0.17938133080800375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,3,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,3,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,7,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,7,0.009514666472872099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,15,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,15,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,31,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,63,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,127,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,127,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,255,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,255,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,511,0.016496000190575916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,511,0.01658133293191592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,2047,0.029658667743206024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,1023,0.01777600000301997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,1023,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,2047,0.02958933264017105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,4095,0.04019733270009359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,4095,0.04001066585381826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,8191,0.021935999393463135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,8191,0.02237333357334137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,16383,0.024256000916163128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,16383,0.026522666215896606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,32767,0.033589333295822144
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,32767,0.030229332546393078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,float16,65535,0.037615999579429626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,1,0.009328000247478485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,1,0.009306666751702627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,1,128,1,float16,fp8,65535,0.042949333786964417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,3,0.009248000259200731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,3,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,7,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,15,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,15,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,63,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,31,0.010255999863147736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,31,0.0102613332370917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,127,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,63,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,127,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,255,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,511,0.016565332810084026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,1023,0.017632000148296356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,255,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,1023,0.01758933315674464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,511,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,2047,0.02992533395687739
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,4095,0.01321600005030632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,4095,0.015583999454975128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,2047,0.029472000896930695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,8191,0.0186666672428449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,8191,0.01595199977358182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,16383,0.020666666328907013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,32767,0.025077333052953083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,16383,0.01899733394384384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,32767,0.024266667664051056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,1,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,1,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,3,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,fp8,65535,0.030762667457262676
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,2,128,1,float16,float16,65535,0.030554667115211487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,3,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,7,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,7,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,15,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,31,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,15,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,31,0.010346666599313417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,63,0.011792000383138657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,63,0.012122667084137598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,127,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,127,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,255,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,255,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,511,0.016490666816631954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,1023,0.017610666652520496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,511,0.016607999801635742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,1023,0.017632000148296356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,2047,0.014954666296641031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,4095,0.015605332950750986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,2047,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,4095,0.010778666784365972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,8191,0.01924266666173935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,16383,0.015583999454975128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,16383,0.021114667256673176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,8191,0.012997332960367203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,32767,0.033999999364217125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,32767,0.022287999590237934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,float16,65535,0.0462666650613149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,1,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,4,128,1,float16,fp8,65535,0.02957333376010259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,3,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,7,0.009317333499590555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,3,0.009642666826645533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,15,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,63,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,31,0.010069333637754122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,63,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,127,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,1,0.009829333052039146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,127,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,255,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,255,0.012106666962305704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,1023,0.013343999783198038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,511,0.016480000068744022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,511,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,1023,0.008053333188096682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,2047,0.015008000036080679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,2047,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,4095,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,4095,0.010725333044926325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,8191,0.02550933261712392
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,8191,0.014266667266686758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,16383,0.02993600070476532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,16383,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,32767,0.04821333289146423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,32767,0.02446399877468745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,float16,65535,0.07221866647402446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,1,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,32,8,128,1,float16,fp8,65535,0.0468746672074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,1,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,3,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,7,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,3,0.009546666716535887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,15,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,15,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,31,0.010373333469033241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,63,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,63,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,255,0.012138667205969492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,255,0.011930666863918304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,127,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,127,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,511,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,511,0.01682666689157486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,2047,0.03746666759252548
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,1023,0.02974933385848999
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,1023,0.029557332396507263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,2047,0.03716266651948293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,4095,0.019013332823912304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,4095,0.017898666361967724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,16383,0.025034666061401367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,8191,0.023370665808518726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,8191,0.021664001047611237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,16383,0.02659733345111211
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,32767,0.03196800003449122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,32767,0.033973333736260734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,float16,65535,0.040607998768488564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,1,128,1,float16,fp8,65535,0.043391997615496315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,1,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,1,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,7,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,3,0.009248000259200731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,3,0.009663999701539675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,7,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,15,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,15,0.009626666704813639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,31,0.01028266673286756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,31,0.010288000106811523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,63,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,127,0.012266666938861212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,127,0.012063999970753988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,63,0.012015999605258306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,255,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,511,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,255,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,511,0.016575999557971954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,1023,0.02957333376010259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,1023,0.029669334491093952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,2047,0.015413332730531693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,2047,0.01137599969903628
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,4095,0.016549333930015564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,4095,0.013066666821638743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,8191,0.01993600030740102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,8191,0.015978666643301647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,16383,0.019141333798567455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,16383,0.022133332987626392
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,32767,0.03562666724125544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,32767,0.02922666569550832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,fp8,65535,0.03754666695992152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,2,128,1,float16,float16,65535,0.04842133323351542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,1,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,3,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,1,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,3,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,7,0.009354666496316591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,15,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,7,0.00985599992175897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,15,0.01003200002014637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,31,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,63,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,63,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,31,0.010341333225369453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,127,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,127,0.012106666962305704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,255,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,255,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,511,0.016965333372354507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,511,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,1023,0.013455999394257864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,1023,0.008389333263039589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,2047,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,2047,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,4095,0.01687466725707054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,4095,0.011002667248249054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,8191,0.015050667027632395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,8191,0.025994665920734406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,16383,0.03107733279466629
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,16383,0.01820266619324684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,32767,0.026533332963784535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,32767,0.04896533489227295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,float16,65535,0.07388266424338023
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,1,0.009455999980370203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,3,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,4,128,1,float16,fp8,65535,0.0488319993019104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,3,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,1,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,7,0.009423999736706415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,7,0.009706666693091393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,31,0.010165333126982054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,15,0.009904000287254652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,31,0.010602666685978571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,63,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,127,0.011962667107582092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,127,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,15,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,63,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,255,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,1023,0.013301332791646322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,255,0.012256000190973282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,511,0.007621333623925845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,511,0.012815999488035837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,2047,0.010362666721145311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,1023,0.008442666381597519
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,2047,0.0204373337328434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,4095,0.023584000766277313
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,4095,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,8191,0.034304000437259674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,8191,0.01618133361140887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,16383,0.05041066805521647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,16383,0.027903998891512554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,32767,0.07833066582679749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,32767,0.03969600051641464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,float16,65535,0.12410133083661397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,1,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,1,0.02607999990383784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,3,0.026746665438016255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,32,8,128,1,float16,fp8,65535,0.06742933392524719
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,3,0.027061333258946735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,7,0.027653334041436512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,7,0.027855999767780304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,15,0.03463999927043915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,15,0.03462400039037069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,31,0.035088000198205314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,31,0.03472000112136205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,63,0.0351946676770846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,63,0.03497066597143809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,127,0.012383999923865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,127,0.009898666913310686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,255,0.016037333756685257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,255,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,511,0.023669332265853882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,1023,0.03402666747570038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,2047,0.054469332098960876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,1023,0.0415040006240209
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,511,0.022448000808556873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,2047,0.05871999760468801
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,4095,0.08541333675384521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,4095,0.09918399651845296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,1,0.025685332715511322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,float16,8191,0.10865066448847453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,1,128,1,float16,fp8,8191,0.13487466176350912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,3,0.02677333354949951
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,1,0.026426665484905243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,3,0.02697066714366277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,7,0.027514666318893433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,7,0.027679999669392902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,15,0.03457066665093104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,31,0.03487999985615412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,15,0.034485332667827606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,31,0.03487999985615412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,63,0.03535466641187668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,63,0.03495999922355016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,127,0.011221333096424738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,127,0.008394666636983553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,255,0.010464000205198923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,255,0.02165866643190384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,511,0.028677334388097126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,511,0.0185759998857975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,1023,0.032501332461833954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,1023,0.04653333127498627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,2047,0.06594133377075195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,2047,0.041589332123597465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,4095,0.08456533153851827
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,4095,0.05942933261394501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,float16,8191,0.12140267093976338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,1,0.02584533393383026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,1,0.02624533325433731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,2,128,1,float16,fp8,8191,0.08741333087285359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,3,0.026778665681680042
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,7,0.027690666417280834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,3,0.026954665780067444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,7,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,15,0.03482133398453394
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,15,0.03454933315515518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,31,0.0351946676770846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,63,0.03522666543722153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,31,0.034901333351929985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,63,0.034901333351929985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,127,0.016048000504573185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,127,0.007402666533986728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,511,0.02145066608985265
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,255,0.03009066730737686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,255,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,511,0.046341334780057274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,1023,0.06155199805895487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,1023,0.027589333554108936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,2047,0.08436266581217448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,4095,0.11964799960454305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,2047,0.04138666639725367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,4095,0.06102400024731954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,float16,8191,0.19183999300003052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,4,128,1,float16,fp8,8191,0.10017066200574239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,1,0.025914666553338368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,1,0.026362667481104534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,3,0.026933332284291584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,3,0.027232001225153606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,7,0.027765333652496338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,7,0.027888000011444092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,15,0.034847999612490334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,15,0.03486400097608566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,31,0.03535466641187668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,31,0.034901333351929985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,63,0.03515733281771342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,63,0.035071998834609985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,127,0.025279998779296875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,255,0.048895999789237976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,255,0.019968000551064808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,511,0.025807999074459076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,511,0.06450133522351582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,1023,0.03818133225043615
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,1023,0.08019733428955078
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,2047,0.11993066469828288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,2047,0.05781333148479462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,4095,0.18990933895111084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,4095,0.09826667110125224
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,float16,8191,0.3296159903208415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,1,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,32,8,128,1,float16,fp8,8191,0.17904533942540488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,1,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,3,0.009519999846816063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,7,0.00997866690158844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,3,0.009706666693091393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,7,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,15,0.009973333527644476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,15,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,31,0.010405333091815313
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,31,0.01073066641887029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,63,0.01227733368674914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,63,0.012229333321253458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,127,0.012074666718641916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,127,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,255,0.012245333443085352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,511,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,255,0.012629333883523941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,511,0.028202667832374573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,1023,0.03251733382542928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,1023,0.032799998919169106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,4095,0.01785600061217944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,4095,0.02088533341884613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,8191,0.025055999557177227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,2047,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,2047,0.015642666568358738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,8191,0.021946666141351063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,16383,0.027215999861558277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,16383,0.026837334036827087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,32767,0.04628799855709076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,float16,65535,0.06280000011126201
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,32767,0.049045334259668984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,1,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,1,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,1,128,1,float16,fp8,65535,0.06601066887378693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,3,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,3,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,7,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,7,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,15,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,31,0.010602666685978571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,15,0.009866666669646898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,63,0.012240000069141388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,31,0.010549332946538925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,127,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,127,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,255,0.012576000144084295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,255,0.01240533341964086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,511,0.02812266598145167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,511,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,1023,0.01369599997997284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,2047,0.01730666682124138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,1023,0.010138666878143946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,2047,0.011616000284751257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,4095,0.018325333793958027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,4095,0.013503999759753546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,8191,0.02716800073782603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,8191,0.018453333526849747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,16383,0.022848000129063923
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,16383,0.03274133304754893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,32767,0.052229334910710655
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,32767,0.04174399872620901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,1,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,fp8,65535,0.05727999905745188
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,2,128,1,float16,float16,65535,0.07621333499749501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,3,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,1,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,7,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,3,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,7,0.0099093330403169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,15,0.009701333319147428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,15,0.010064000263810158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,31,0.010538666198650995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,31,0.010415999839703241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,63,0.012159999459981918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,63,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,127,0.012378666549921036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,127,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,255,0.012517333030700684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,255,0.012245333443085352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,1023,0.013248000293970108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,511,0.00786666696270307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,511,0.012800000607967377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,1023,0.008682666967312494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,2047,0.021850667893886566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,2047,0.010672000547250112
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,4095,0.025274666647116344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,8191,0.03590933233499527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,4095,0.012639999389648438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,16383,0.053957333167394005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,8191,0.01701333373785019
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,32767,0.0801333338022232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,16383,0.028944000601768494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,32767,0.04126933217048645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,fp8,65535,0.06977066894372304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,1,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,1,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,4,128,1,float16,float16,65535,0.12572800119717917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,3,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,3,0.010144000252087912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,7,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,15,0.009882666791478792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,31,0.010410666465759277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,7,0.009824000298976898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,15,0.010181333248813948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,63,0.012191999703645706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,31,0.010581333190202713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,63,0.01227733368674914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,127,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,127,0.012506666282812754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,255,0.012693333129088083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,511,0.007690666864315669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,511,0.013669333110253016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,255,0.007285333548982938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,1023,0.01911466692884763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,1023,0.009039999917149544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,2047,0.028357334434986115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,4095,0.04032533367474874
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,4095,0.018640000373125076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,2047,0.01145600030819575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,8191,0.058287998040517174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,8191,0.024314666787783306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,16383,0.08591467142105103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,16383,0.04181333382924398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,32767,0.12230400244394939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,32767,0.06306133170922597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,1,0.047557334105173744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,float16,65535,0.19808000326156616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,32,8,128,1,float16,fp8,65535,0.10795199871063232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,1,0.04822933177153269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,3,0.04898133377234141
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,3,0.04996799925963084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,7,0.05110399921735128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,7,0.05132266879081726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,15,0.06365866462389629
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,15,0.06392533580462138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,31,0.06443733473618825
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,31,0.06392533580462138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,63,0.06462400158246358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,63,0.06474133332570393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,127,0.013050666699806849
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,127,0.010224000240365664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,255,0.025733334322770435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,255,0.021402666966120403
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,511,0.03475733349720637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,511,0.039936001102129616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,1023,0.058090666929880776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,1023,0.073594664533933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,2047,0.0830506682395935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,2047,0.10718933741251628
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,float16,4095,0.10885866483052571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,1,128,1,float16,fp8,4095,0.13545067111651102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,1,0.047237331668535866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,1,0.04818666477998098
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,3,0.0490880012512207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,3,0.05004799862702688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,7,0.05134933193524679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,7,0.051167999704678856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,15,0.06384533147017162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,15,0.06377066671848297
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,31,0.06424533327420552
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,31,0.06391466657320659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,63,0.06462400158246358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,63,0.06448533137639363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,127,0.016341333587964375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,127,0.008714666590094566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,255,0.031082667410373688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,255,0.01785600061217944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,511,0.04811733464399973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,511,0.03183999905983607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,1023,0.06251200040181477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,1023,0.04125866790612539
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,2047,0.08492799599965413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,2047,0.05931200087070465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,float16,4095,0.12173866232236226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,1,0.04820266862710317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,2,128,1,float16,fp8,4095,0.086709330479304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,3,0.049178664882977806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,3,0.05020266771316528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,7,0.051183998584747314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,1,0.0473333348830541
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,7,0.051632001996040344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,15,0.06393600006898244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,15,0.06379200021425883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,63,0.06448000172773997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,31,0.06449600060780843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,127,0.02607999990383784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,31,0.06391466657320659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,63,0.0646613339583079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,511,0.06453333298365276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,255,0.04946133494377136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,255,0.021664001047611237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,511,0.028245332340399425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,1023,0.041562666495641075
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,1023,0.08102933565775554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,2047,0.12124266227086385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,2047,0.06131733457247416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,1,0.04734399914741516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,fp8,4095,0.10057066877683003
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,4,128,1,float16,float16,4095,0.19315733512242636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,1,0.04851733148097992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,3,0.04926399886608124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,3,0.05022400120894114
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,7,0.05194666484991709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,7,0.05129600067933401
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,15,0.0642133355140686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,15,0.06394133468468984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,31,0.06457599997520447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,63,0.06477866570154826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,31,0.0641653339068095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,63,0.06462400158246358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,127,0.04520000020662943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,127,0.016016000260909397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,255,0.05236800014972687
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,255,0.021312000850836437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,511,0.06935466825962067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,511,0.03372266640265783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,1023,0.10314666231473286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,1023,0.053472002347310386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,2047,0.17620267470677695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,1,0.09011200070381165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,2047,0.09424533446629842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,fp8,4095,0.17502933740615845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,32,8,128,1,float16,float16,4095,0.3103626569112142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,1,0.09160533547401428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,3,0.09324799974759419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,3,0.09423999985059102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,7,0.09724799791971843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,7,0.09781333804130554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,15,0.12203199664751689
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,15,0.12181333700815837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,31,0.12260799606641133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,31,0.12193600336710612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,63,0.12345600128173828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,63,0.12273066242535909
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,127,0.01749333366751671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,127,0.020288000504175823
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,255,0.03868266691764196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,255,0.03899733225504557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,511,0.0603413333495458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,511,0.07258133093516032
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,float16,1023,0.07852800190448761
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,1,128,1,float16,fp8,1023,0.10293333729108174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,1,0.09002133210500081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,3,0.09319466352462769
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,1,0.09193600217501323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,3,0.09427199761072795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,7,0.09717866778373718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,7,0.09809600313504536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,15,0.12219732999801636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,31,0.12268799543380737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,15,0.12210667133331299
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,31,0.12176000078519185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,63,0.123471995194753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,63,0.1225920021533966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,127,0.02720000098148982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,127,0.014597332725922266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,255,0.03160533308982849
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,255,0.05195199946562449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,511,0.06654933094978333
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,511,0.04164266586303711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,float16,1023,0.08310933411121368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,1,0.0902239978313446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,1,0.09172800183296204
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,2,128,1,float16,fp8,1023,0.05973866581916809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,3,0.09322133660316467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,3,0.09458667039871216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,7,0.09724266330401103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,7,0.09819199641545613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,15,0.12226133545239766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,15,0.12219732999801636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,31,0.12271466851234436
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,31,0.12214932839075725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,63,0.12346133589744568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,63,0.12319466471672058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,127,0.017738666385412216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,127,0.046223998069763184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,255,0.05425066749254862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,255,0.024832000335057575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,511,0.07036266724268596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,511,0.0373279998699824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,float16,1023,0.10570666193962097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,4,128,1,float16,fp8,1023,0.05658133327960968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,1,0.0902880032857259
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,1,0.0920853316783905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,3,0.09351467092831929
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,3,0.09482133388519287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,15,0.12221866846084595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,7,0.09751466910044353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,7,0.09866666793823242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,31,0.12294933199882507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,15,0.12264000376065572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,31,0.12261333068211873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,63,0.12959466377894083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,127,0.08303466439247131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,63,0.12339733044306438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,127,0.02924266705910365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,255,0.09366933504740398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,255,0.041493333876132965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,511,0.12662933270136514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,511,0.061493332187334694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,fp8,1023,0.1013866662979126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,3,0.18012799819310507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,1,0.17454934120178223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,3,0.1824693282445272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,32,8,128,1,float16,float16,1023,0.193557341893514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,1,0.17814399798711142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,7,0.18922666708628336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,7,0.19155732790629068
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,15,0.23837866385777792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,31,0.23842666546503702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,15,0.2387626568476359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,31,0.23959465821584067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,63,0.24102399746576944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,127,0.03494933247566223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,63,0.24003199736277261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,127,0.03235200047492981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,255,0.06681600213050842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,255,0.07664533456166585
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,fp8,511,0.11343999703725179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,1,128,1,float16,float16,511,0.08307200173536937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,1,0.17465599377950033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,1,0.17805333932240805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,3,0.18021865685780844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,3,0.1827359994252523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,7,0.1891253391901652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,7,0.19119999806086221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,15,0.23847466707229614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,15,0.23881600300470987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,31,0.24002132813135782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,31,0.23878933986028036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,63,0.2409706711769104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,127,0.0490880012512207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,63,0.24023467302322388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,127,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,255,0.056464001536369324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,255,0.037061333656311035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,float16,511,0.07387733459472656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,1,0.17810134092966715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,1,0.17457600434621176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,2,128,1,float16,fp8,511,0.05514666438102722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,3,0.1805866758028666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,3,0.18288532892862955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,7,0.18937599658966064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,7,0.19164266188939413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,15,0.23884799083073935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,15,0.23920534054438272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,31,0.24032533168792725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,31,0.23915199438730875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,63,0.2502773404121399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,63,0.24057066440582275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,127,0.0860746701558431
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,255,0.09774933258692424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,127,0.03334933271010717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,255,0.04724800089995066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,float16,511,0.12924266854921976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,1,0.17484800020853677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,4,128,1,float16,fp8,511,0.06670933465162913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,1,0.17818133036295572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,3,0.18076266845067343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,7,0.1922880013783773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,3,0.18365333477656046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,7,0.18968532482783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,15,0.24011200666427612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,15,0.23988799254099527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,31,0.24043200413386026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,31,0.24986666440963745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,63,0.252895991007487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,63,0.2521546681722005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,127,0.15422399838765463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,127,0.05675200124581655
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,1,0.008986666798591614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,255,0.1759200096130371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,255,0.07533866663773854
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,float16,511,0.23758933941523233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,32,8,128,1,float16,fp8,511,0.11382399996121724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,7,0.009237333511312803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,3,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,3,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,7,0.00938666673998038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,31,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,15,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,15,0.009898666913310686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,63,0.011829332758982977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,31,0.011648000528415045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,63,0.012053333222866058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,127,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,127,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,511,0.018981333822011948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,511,0.019205333044131596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,255,0.01360000049074491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,255,0.013690666606028875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,2047,0.020031999796628952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,1023,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,1023,0.013738666971524557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,2047,0.015578666081031164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,4095,0.018229333062966663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,4095,0.02124800036350886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,8191,0.033301333586374916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,8191,0.03243733445803324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,16383,0.03917866696914037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,16383,0.03972800076007843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,32767,0.06515733400980632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,32767,0.07061866422494252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,float16,65535,0.09914132952690125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,1,0.009050666665037474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,1,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,3,0.008965333302815756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,1,128,1,float16,fp8,65535,0.11079466342926025
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,3,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,7,0.009178666397929192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,15,0.009941333283980688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,7,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,15,0.010181333248813948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,63,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,31,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,31,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,63,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,127,0.011770666887362799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,511,0.012741333494583765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,127,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,255,0.013738666971524557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,255,0.013514666507641474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,1023,0.013381333400805792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,511,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,1023,0.010399999717871347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,2047,0.02250666668017705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,2047,0.013061333447694778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,4095,0.025445332129796345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,4095,0.015493333339691162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,8191,0.02646933247645696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,16383,0.053690666953722634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,8191,0.03661333272854487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,16383,0.0340639998515447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,32767,0.08125866452852885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,32767,0.060319999853769936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,1,0.008778666456540426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,1,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,fp8,65535,0.09583466251691182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,2,128,1,float16,float16,65535,0.1281706690788269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,3,0.009301333377758661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,3,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,7,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,7,0.009258666386206945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,15,0.009866666669646898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,15,0.009904000287254652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,31,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,63,0.011621333658695221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,63,0.011813333878914515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,31,0.01190399999419848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,127,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,255,0.012554666648308435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,255,0.007642666498819987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,511,0.013434667140245438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,2047,0.02869333326816559
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,511,0.008240000034372011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,1023,0.019018666197856266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,1023,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,2047,0.012309333930412928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,4095,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,4095,0.04016000032424927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,8191,0.026346666117509205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,8191,0.05865600208441416
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,16383,0.08666666348775227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,16383,0.044906665881474815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,32767,0.06647466619809468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,32767,0.12358933687210083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,1,0.009002666920423508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,1,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,3,0.009141333401203156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,fp8,65535,0.10973866780598958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,3,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,4,128,1,float16,float16,65535,0.1992959976196289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,7,0.009290666629870733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,7,0.009322666873534521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,15,0.010053333515922228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,15,0.010234666367371878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,63,0.011765333513418833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,31,0.011557333171367645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,31,0.01180800050497055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,63,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,127,0.010304000228643417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,127,0.006469333544373512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,255,0.013258667041858038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,511,0.01951466624935468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,1023,0.027962667246659596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,511,0.008623999853928884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,255,0.007584000006318092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,1023,0.014181333283583323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,4095,0.06680533289909363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,2047,0.042591998974482216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,2047,0.01735466718673706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,4095,0.02681066592534383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,8191,0.08404800295829773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,8191,0.03967999915281931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,16383,0.0588319996992747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,16383,0.11876799662907918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,32767,0.18998400370279947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,32767,0.10008000334103902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,float16,65535,0.32972800731658936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,32,8,128,1,float16,fp8,65535,0.18059200048446655
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,1,0.3442773421605428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,7,0.3722720146179199
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,1,0.3508799870808919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,3,0.3558613459269206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,3,0.36125866572062176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,7,0.3766239881515503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,15,0.4713706572850545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,15,0.4724213282267253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,31,0.4734613498051961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,31,0.47172800699869794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,63,0.4785439968109131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,63,0.4748053153355916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,127,0.06529599924882253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,127,0.06490666667620341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,float16,255,0.07366399963696797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,1,128,1,float16,fp8,255,0.10392533739407857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,1,0.34426132837931317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,1,0.3506079912185669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,3,0.35571734110514325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,7,0.3726826508839925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,3,0.361077348391215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,7,0.3769013484319051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,15,0.47197866439819336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,15,0.4726346731185913
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,31,0.4779413143793742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,31,0.4721386830012004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,63,0.4894346793492635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,63,0.48013333479563397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,127,0.08990933497746785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,127,0.06374933322270711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,float16,255,0.1025333305199941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,2,128,1,float16,fp8,255,0.0775786687930425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,1,0.3440693219502767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,1,0.35096001625061035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,3,0.35606932640075684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,3,0.36105600992838544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,7,0.37402133146921795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,7,0.3777120113372803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,15,0.47672533988952637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,15,0.4737066825230916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,31,0.4883519808451335
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,31,0.47994665304819745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,63,0.4926613171895345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,63,0.4928640127182007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,127,0.16115199526151022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,127,0.06603200236956279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,fp8,255,0.08447999755541484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,1,0.3444799979527791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,4,128,1,float16,float16,255,0.1832853356997172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,1,0.351151982943217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,3,0.36185065905253094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,7,0.3867093324661255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,7,0.3816266854604085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,3,0.35783998171488446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,15,0.48571733633677167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,15,0.4825013478597005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,31,0.4939306577046712
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,31,0.492847998936971
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,63,0.4975839853286743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,63,0.49836266040802
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,127,0.2978079915046692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,127,0.10683733224868774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,float16,255,0.34282131989796955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,32,8,128,1,float16,fp8,255,0.1434879998366038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,1,0.6858133474985758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,1,0.7003413041432699
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,3,0.7148213386535645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,7,0.7491947015126547
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,3,0.7208853562672933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,7,0.7533600330352783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,15,0.9428266684214274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,15,0.9458186626434326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,31,0.9589813550313314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,31,0.9504746596018473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,63,0.965936024983724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,63,0.9662400086720785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,float16,127,0.12154133121172588
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,1,128,1,float16,fp8,127,0.1781866749127706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,1,0.6887679894765218
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,1,0.7006879647572836
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,3,0.7193013032277426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,3,0.7227733135223389
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,7,0.7636853059132894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,7,0.7642666498819987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,15,0.9529333114624023
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,15,0.9519840081532797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,31,0.9634666442871094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,31,0.9620587031046549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,63,0.9715519746144613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,63,0.9726239840189616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,fp8,127,0.12333333492279053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,2,128,1,float16,float16,127,0.16858667135238647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,1,0.7026293277740479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,1,0.7098986307779948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,3,0.7345653374989828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,3,0.7379946708679199
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,7,0.7735733191172282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,7,0.781872034072876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,15,0.9575520356496176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,15,0.9628053506215414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,31,0.9698826471964518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,31,0.9692800045013428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,63,0.9777920246124268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,63,0.9788266817728678
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,float16,127,0.3108479976654053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,4,128,1,float16,fp8,127,0.12329600254694621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,1,0.7254933516184489
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,1,0.7365067005157471
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,3,0.7469440301259359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,3,0.7585759957631429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,7,0.7816960016886393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,7,0.7950186729431152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,15,0.9645067056020101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,15,0.9726080099741617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,31,0.9803199768066406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,31,0.9794933001200358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,63,0.9881813526153564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,63,0.9889439741770426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,1,0.014858666807413101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,float16,127,0.5816160043080648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,1,0.015125333021084467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,3,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,32,8,128,1,float16,fp8,127,0.20684266090393066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,3,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,7,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,15,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,7,0.015381333728631338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,31,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,15,0.016549333930015564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,31,0.019866666446129482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,63,0.020181333025296528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,63,0.020069333414236706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,127,0.02006400004029274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,127,0.020165332903464634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,255,0.023599999646345775
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,255,0.023578666150569916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,511,0.015034666905800501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,511,0.012725333372751871
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,2047,0.025648000339667004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,1023,0.013770667215188345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,1023,0.015728000551462173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,2047,0.024154665569464367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,4095,0.027914665639400482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,4095,0.030080000559488933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,8191,0.04637333254019419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,8191,0.04438399771849314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,16383,0.06513066589832306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,16383,0.07274666428565979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,float16,32767,0.10518399874369304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,1,0.014837333311637243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,1,128,1,float16,fp8,32767,0.12716266512870789
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,3,0.014922666052977243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,3,0.015450666348139444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,7,0.015135999768972397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,7,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,15,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,15,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,31,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,31,0.0199946661790212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,63,0.01993600030740102
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,63,0.020031999796628952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,255,0.012634667257467905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,127,0.02011200040578842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,127,0.020080000162124634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,255,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,511,0.013424000392357508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,511,0.00960533320903778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,1023,0.019237333287795384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,1023,0.011962667107582092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,2047,0.02922666569550832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,2047,0.020293333878119785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,4095,0.040752001106739044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,4095,0.023946667710940044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,8191,0.05926933387915293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,8191,0.039834665755430855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,16383,0.08700266480445862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,16383,0.06294399996598561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,1,0.01479999969402949
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,float16,32767,0.12435199817021687
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,2,128,1,float16,fp8,32767,0.09803733229637146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,1,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,3,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,3,0.015402667224407196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,7,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,15,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,7,0.015573333948850632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,15,0.01655999943614006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,31,0.01985599969824155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,31,0.020037333170572918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,63,0.019941333681344986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,63,0.019999999552965164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,127,0.010277333358923594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,127,0.008037333066264788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,255,0.007994666695594788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,255,0.013642666240533194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,511,0.01916266605257988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,511,0.009354666496316591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,1023,0.027674667537212372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,1023,0.015301333119471868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,2047,0.04304533203442892
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,2047,0.018944000204404194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,4095,0.06692266464233398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,4095,0.02888533224662145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,8191,0.08505599697430928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,8191,0.04271999994913737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,1,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,16383,0.11993599931399028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,16383,0.0611413319905599
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,3,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,fp8,32767,0.1013813316822052
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,4,128,1,float16,float16,32767,0.19074666500091553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,3,0.015594666202863058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,7,0.015018666783968607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,15,0.016597333053747814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,7,0.015509333461523056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,15,0.016399999459584553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,31,0.01985599969824155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,63,0.020207999895016353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,31,0.020026666422684986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,63,0.020101333657900494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,127,0.010522666076819101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,127,0.007029333462317784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,255,0.008336000144481659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,255,0.020645332833131153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,511,0.028160000840822857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,511,0.01369599997997284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,1023,0.04517333209514618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,1023,0.02038399999340375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,2047,0.06516799827416737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,2047,0.025759999950726826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,4095,0.0830506682395935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,4095,0.03868266691764196
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,8191,0.11934399604797363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,8191,0.06058133145173391
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,16383,0.18817599614461264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,16383,0.09854933619499207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,float16,32767,0.3287946581840515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,32,8,128,1,float16,fp8,32767,0.18004266421000162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,3,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,1,0.015098666151364645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,1,0.014789332946141561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,3,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,15,0.016544000556071598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,7,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,7,0.015418666104475657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,31,0.0198186660806338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,15,0.016602666427691776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,31,0.019925333559513092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,63,0.019930666933457058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,63,0.020128000527620316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,127,0.020026666422684986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,127,0.020181333025296528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,255,0.012757333616415659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,255,0.009141333401203156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,511,0.013482666263977686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,511,0.009674666449427605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,1023,0.011776000261306763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,1023,0.01926933353145917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,2047,0.02922133356332779
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,4095,0.023818666736284893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,2047,0.019797333826621372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,4095,0.04267199834187826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,8191,0.05982399980227152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,8191,0.03991466760635376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,16383,0.08899199962615967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,16383,0.06281599899133046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,1,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,3,0.015002666662136713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,1,0.015354666858911514
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,float16,32767,0.12517866492271423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,1,128,1,float16,fp8,32767,0.09753599762916565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,3,0.015311999867359797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,7,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,7,0.01545599972208341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,15,0.01657066618402799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,15,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,31,0.0199946661790212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,31,0.019930666933457058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,63,0.019920000185569126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,63,0.02004266654451688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,127,0.01033599985142549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,127,0.0069333333522081375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,255,0.013376000026861826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,255,0.007983999947706858
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,511,0.009973333527644476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,511,0.019567999988794327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,1023,0.02775999903678894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,2047,0.04346133271853129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,1023,0.015301333119471868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,2047,0.018800000349680584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,4095,0.06698666512966156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,4095,0.029205332199732464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,8191,0.0848586658636729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,8191,0.042378668983777366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,16383,0.12103999654452006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,16383,0.06132266422112783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,1,0.015103999525308609
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,float16,32767,0.19223999977111816
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,3,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,1,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,7,0.015024000157912573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,3,0.015754666179418564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,2,128,1,float16,fp8,32767,0.10150399804115295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,15,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,7,0.015669333438078564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,31,0.019999999552965164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,15,0.016938666502634685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,31,0.019893333315849304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,63,0.019871999820073444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,63,0.020213333268960316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,127,0.010741333166758219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,127,0.0068853336075941724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,255,0.008378666515151659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,255,0.02073066681623459
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,511,0.02792533238728841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,1023,0.045408000548680626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,511,0.01349866638580958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,1023,0.020128000527620316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,2047,0.06528533498446147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,2047,0.025573333104451496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,4095,0.08343467116355896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,4095,0.03862933317820231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,8191,0.06036800146102905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,8191,0.11989866693814595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,16383,0.18928533792495728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,16383,0.09847999612490337
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,float16,32767,0.3314826687177022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,4,128,1,float16,fp8,32767,0.17945599555969238
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,1,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,3,0.015541333705186844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,7,0.015685333559910457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,15,0.016719999412695568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,31,0.02015999952952067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,63,0.01998399943113327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,127,0.007376000285148621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,1,0.014874666929244995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,127,0.015376000354687372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,3,0.015200000256299973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,7,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,31,0.019765333582957584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,511,0.04590400060017904
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,255,0.030005333324273426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,15,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,1023,0.06113066772619883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,63,0.02021866664290428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,2047,0.03809600075085958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,2047,0.08354666829109192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,1023,0.025231999655564625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,255,0.013093333691358566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,4095,0.11838400363922119
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,8191,0.18954133987426758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,511,0.019600000232458115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,8191,0.0976746678352356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,16383,0.3296479980150859
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,16383,0.1785866618156433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,1,0.009194666519761086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,1,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,float16,32767,0.60807998975118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,4095,0.05825066566467285
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,3,0.009093333035707474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,7,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,3,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,7,0.009205333267649015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,15,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,15,0.009503999724984169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,31,0.009930666536092758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,31,0.010144000252087912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,16,8,128,1,float16,fp8,32767,0.34239999453226727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,127,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,63,0.011861333002646765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,63,0.011781333635250727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,127,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,255,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,511,0.016384000579516094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,255,0.011776000261306763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,511,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,1023,0.01762666677435239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,1023,0.017509333789348602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,2047,0.017759999881188076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,4095,0.029738667110602062
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,2047,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,4095,0.029509333272775013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,8191,0.01794133335351944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,8191,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,16383,0.019839999576409657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,32767,0.02407466620206833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,32767,0.023936000963052113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,16383,0.018810667097568512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,65535,0.029157333076000214
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,65535,0.03027733415365219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,1,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,1,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,3,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,3,0.009141333401203156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,float16,131071,0.039834665755430855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,1,128,1,float16,fp8,131071,0.04025600105524063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,7,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,7,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,15,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,15,0.009717333440979322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,31,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,31,0.010202666744589806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,63,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,127,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,127,0.011584000041087469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,63,0.011648000528415045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,255,0.011930666863918304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,511,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,511,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,255,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,1023,0.017573333034912746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,2047,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,1023,0.017301333447297413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,2047,0.01775466650724411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,4095,0.015040000279744467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,4095,0.010794666906197866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,8191,0.018464000274737675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,8191,0.012693333129088083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,16383,0.020143999407688778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,16383,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,32767,0.024160000185171764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,32767,0.01937066639463107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,65535,0.029813334345817566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,65535,0.024127999941507976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,1,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,1,0.009226666763424873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,float16,131071,0.057461331288019814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,3,0.009359999870260557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,3,0.00897066667675972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,2,128,1,float16,fp8,131071,0.03585600107908249
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,7,0.008949333180983862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,7,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,15,0.00938666673998038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,15,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,31,0.010213333492477735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,31,0.01022933361430963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,63,0.01202133297920227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,127,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,127,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,63,0.01180800050497055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,255,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,511,0.016373333831628162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,255,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,2047,0.014677333335081736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,511,0.01639466608564059
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,1023,0.01754133279124896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,1023,0.017386666188637417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,2047,0.008954666554927826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,4095,0.010165333126982054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,4095,0.015669333438078564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,8191,0.012383999923865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,16383,0.014901333798964819
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,8191,0.019354666272799175
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,16383,0.021002667645613354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,32767,0.03409066547950109
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,32767,0.022128000855445862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,1,0.009354666496316591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,65535,0.04586666822433472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,float16,131071,0.07628266513347626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,1,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,65535,0.028416000306606293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,3,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,4,128,1,float16,fp8,131071,0.042165334026018776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,3,0.009248000259200731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,15,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,7,0.009466666728258133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,7,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,15,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,63,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,31,0.010069333637754122
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,31,0.01009599988659223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,63,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,127,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,127,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,511,0.016389333953460056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,255,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,511,0.01632533346613248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,1023,0.007957333077987036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,255,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,1023,0.013434667140245438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,2047,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,4095,0.01580799991885821
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,8191,0.025455998877684276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,2047,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,8191,0.014368000129858652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,16383,0.03033600002527237
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,4095,0.010357333347201347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,16383,0.017488000293572743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,32767,0.04780266682306925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,32767,0.02513599892457326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,65535,0.07189866900444031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,65535,0.046581332882245384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,1,0.009370666618148485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,float16,131071,0.123471995194753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,1,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,16,8,128,1,float16,fp8,131071,0.06676266590754192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,3,0.009237333511312803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,3,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,7,0.009466666728258133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,7,0.009359999870260557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,15,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,15,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,31,0.010421333213647207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,31,0.010245333115259806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,63,0.011866666376590729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,63,0.011871999750534693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,127,0.012058666596810022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,127,0.012026666353146235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,255,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,255,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,511,0.01659199967980385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,1023,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,511,0.016677333662907284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,1023,0.017562666287024815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,2047,0.029685333371162415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,4095,0.015962666521469753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,2047,0.029743999242782593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,4095,0.01314666618903478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,8191,0.019205333044131596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,8191,0.015728000551462173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,16383,0.01897066707412402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,16383,0.020842666427294414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,32767,0.024117333193620045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,32767,0.02532800038655599
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,65535,0.031082667410373688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,65535,0.03143466760714849
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,1,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,float16,131071,0.059658666451772056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,3,0.009178666397929192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,1,0.009701333319147428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,3,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,1,128,1,float16,fp8,131071,0.04864533245563507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,7,0.009130666653315226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,7,0.009663999701539675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,15,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,15,0.009786666681369146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,31,0.010186666622757912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,63,0.01191466674208641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,63,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,127,0.01192533348997434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,31,0.010357333347201347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,255,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,127,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,255,0.012037333101034164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,511,0.016442666451136272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,511,0.01640533283352852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,1023,0.017594666530688603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,1023,0.0176959993938605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,2047,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,2047,0.015791999797026317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,4095,0.01071999967098236
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,4095,0.016399999459584553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,8191,0.020037333170572918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,8191,0.012869333227475485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,16383,0.015429332852363586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,32767,0.035088000198205314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,16383,0.02164799968401591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,32767,0.021744000415007275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,65535,0.04741866886615753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,65535,0.02942399928967158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,float16,131071,0.07719466586907704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,1,0.009194666519761086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,1,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,2,128,1,float16,fp8,131071,0.04346133271853129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,3,0.009162666896979014
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,7,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,7,0.009653333574533463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,3,0.009632000078757605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,15,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,15,0.009450666606426239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,31,0.010138666878143946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,63,0.012042666474978128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,31,0.010410666465759277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,127,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,63,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,255,0.012122667084137598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,127,0.012037333101034164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,255,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,511,0.0162773331006368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,511,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,1023,0.0081386665503184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,2047,0.01586666703224182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,1023,0.013381333400805792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,2047,0.008885333314538002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,4095,0.010319999729593595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,4095,0.016565332810084026
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,8191,0.014181333283583323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,8191,0.025770666698614757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,16383,0.0308746670683225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,16383,0.01749333366751671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,32767,0.02476799984773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,32767,0.04937066634496053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,65535,0.07326933244864146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,65535,0.04699199895064036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,1,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,1,0.009839999799927076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,fp8,131071,0.0665280024210612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,3,0.009237333511312803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,4,128,1,float16,float16,131071,0.124917338291804
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,3,0.009824000298976898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,7,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,15,0.00979200005531311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,7,0.00956266683836778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,15,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,31,0.010239999741315842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,63,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,31,0.010357333347201347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,127,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,63,0.011866666376590729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,127,0.012037333101034164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,511,0.012800000607967377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,1023,0.013434667140245438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,255,0.01209066684047381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,255,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,511,0.007701333612203598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,1023,0.008256000156203905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,2047,0.020586666961510975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,4095,0.023754666248957317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,2047,0.010170666500926018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,4095,0.011781333635250727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,16383,0.027973333994547527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,16383,0.05134933193524679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,8191,0.03514133393764496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,8191,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,32767,0.03938666731119156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,32767,0.07868266602357228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,65535,0.12437867124875386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,65535,0.06717333197593689
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,1,0.014688000082969666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,1,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,float16,131071,0.20728532473246256
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,16,8,128,1,float16,fp8,131071,0.11198400457700093
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,3,0.015173333386580149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,3,0.015210667004187902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,7,0.015850666910409927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,7,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,15,0.0194560003777345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,15,0.019194666296243668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,31,0.019482667247454327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,31,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,63,0.019551999866962433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,63,0.01941866676012675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,127,0.010480000327030817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,255,0.013621332744757334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,255,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,127,0.00821333316465219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,511,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,511,0.010863999525705973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,1023,0.027999999622503918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,1023,0.019674666225910187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,2047,0.04481600224971771
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,2047,0.02757333219051361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,4095,0.044826666514078774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,4095,0.0682239979505539
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,8191,0.06003733476003011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,8191,0.08615466952323914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,1,0.014805333067973455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,3,0.01526933287580808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,float16,16383,0.12304000059763591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,1,0.015109332899252573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,3,0.015477333217859268
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,1,128,1,float16,fp8,16383,0.08781333764394124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,7,0.015754666179418564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,7,0.015781333049138386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,15,0.01937066639463107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,15,0.01956266661485036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,31,0.01945066700379054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,31,0.019530666371186573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,127,0.010944000134865442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,63,0.01966933285196622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,63,0.019487999379634857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,127,0.007269333427151044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,255,0.02093333254257838
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,255,0.009045333291093508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,511,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,511,0.015072000523408255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,1023,0.04545066754023234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,2047,0.06516799827416737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,2047,0.02752533306678136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,1023,0.02164799968401591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,4095,0.08430400490760803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,4095,0.04127466678619385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,8191,0.12011200189590454
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,8191,0.06316266457239787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,1,0.01482133318980535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,float16,16383,0.19152534008026123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,1,0.015157333264748255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,3,0.015189333508412043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,2,128,1,float16,fp8,16383,0.10107733805974324
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,3,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,7,0.015925332903862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,7,0.016224000602960587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,15,0.01953599974513054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,31,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,31,0.01953599974513054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,15,0.019413333386182785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,63,0.01966399947802226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,255,0.013264000415802002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,63,0.01974933346112569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,255,0.029685333371162415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,127,0.007242666557431221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,511,0.04598399996757507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,127,0.015562667200962702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,511,0.01987733319401741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,1023,0.06079466640949249
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,1023,0.025519999365011852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,2047,0.03836799909671148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,2047,0.08338133494059245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,4095,0.11947733163833618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,4095,0.05852800110975901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,8191,0.1900426745414734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,8191,0.09763200084368388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,float16,16383,0.3319946726163228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,1,0.01469333345691363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,1,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,4,128,1,float16,fp8,16383,0.1790133317311605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,3,0.015482666591803232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,3,0.015450666348139444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,7,0.01584533353646596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,7,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,15,0.019546666493018467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,15,0.019386666516462963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,31,0.019391999890406925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,63,0.019706666469573975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,31,0.019551999866962433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,63,0.019760000209013622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,127,0.025466665625572205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,127,0.010794666906197866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,255,0.04864533245563507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,255,0.019482667247454327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,511,0.06438399851322174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,511,0.025279998779296875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,1023,0.08041599889596303
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,1023,0.03825599948565165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,2047,0.11929066975911458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,2047,0.05725333094596863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,4095,0.09698667128880818
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,4095,0.18998400370279947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,8191,0.32977600892384845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,8191,0.17815999190012613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,1,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,1,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,float16,16383,0.6107466618220011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,3,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,16,8,128,1,float16,fp8,16383,0.340773344039917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,3,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,7,0.009392000113924345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,15,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,7,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,31,0.01028266673286756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,15,0.009824000298976898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,63,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,31,0.010458666831254959
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,63,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,255,0.012266666938861212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,127,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,127,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,255,0.01231466606259346
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,511,0.01685333376129468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,511,0.016650666793187458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,1023,0.02974933385848999
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,1023,0.029792000850041706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,2047,0.016586666305859882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,4095,0.013162666310866674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,4095,0.01788266624013583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,2047,0.011440000186363855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,8191,0.021509334444999695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,8191,0.01575999955336253
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,16383,0.023498666783173878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,16383,0.019167999426523846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,32767,0.037690666814645134
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,65535,0.03771200031042099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,32767,0.028512001037597656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,65535,0.05030933519204458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,1,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,1,0.009653333574533463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,fp8,131071,0.070933332045873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,1,128,1,float16,float16,131071,0.08110933502515157
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,3,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,3,0.009317333499590555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,7,0.009418666362762451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,7,0.009653333574533463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,15,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,15,0.009925333162148794
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,31,0.010224000240365664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,63,0.012122667084137598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,63,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,31,0.010426666587591171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,127,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,127,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,255,0.012383999923865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,255,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,511,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,511,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,1023,0.013365333278973898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,2047,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,1023,0.008565333361426989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,2047,0.01706133286158244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,4095,0.018160000443458557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,8191,0.027349332968393963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,16383,0.03202133377393087
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,4095,0.011109333485364914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,8191,0.014864000181357065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,16383,0.018239999810854595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,32767,0.026501332720120747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,32767,0.05136000116666158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,65535,0.05017066498597463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,65535,0.07549866537253062
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,1,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,float16,131071,0.12709333499272665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,2,128,1,float16,fp8,131071,0.06856533388296764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,1,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,3,0.009237333511312803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,7,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,3,0.009877333417534828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,15,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,7,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,15,0.009989333028594652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,31,0.010426666587591171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,63,0.01202133297920227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,63,0.012159999459981918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,255,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,127,0.012250666817029318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,127,0.012341332932313284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,511,0.012661332885424295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,511,0.007653333246707916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,255,0.012538666526476542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,1023,0.013354666531085968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,1023,0.008197333042820295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,2047,0.02201066662867864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,2047,0.010202666744589806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,4095,0.012063999970753988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,4095,0.02492266645034154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,8191,0.03626666714747747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,8191,0.01621866722901662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,16383,0.027802666028340656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,16383,0.053727999329566956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,32767,0.08056533336639404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,32767,0.039349332451820374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,65535,0.12550933162371317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,65535,0.06684799989064534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,1,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,1,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,float16,131071,0.21015467246373495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,3,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,3,0.009722666814923286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,4,128,1,float16,fp8,131071,0.11182933052380879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,15,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,7,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,7,0.009690666571259499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,15,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,31,0.010549332946538925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,127,0.0120319997270902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,63,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,127,0.012069333344697952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,63,0.012261333564917246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,255,0.012426666915416718
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,255,0.007226666435599327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,511,0.008218666538596153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,511,0.013183999806642532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,1023,0.019098666807015736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,1023,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,2047,0.0283146674434344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,2047,0.011322667201360067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,4095,0.041178666055202484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,4095,0.018613333503405254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,8191,0.058592001597086586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,8191,0.02439466615517934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,16383,0.08714666962623596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,16383,0.041850666205088295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,32767,0.12274133165677388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,65535,0.1980266571044922
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,32767,0.06326400240262349
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,65535,0.10800533493359883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,1,0.02589333305756251
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,float16,131071,0.35478933652242023
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,16,8,128,1,float16,fp8,131071,0.19395732879638672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,1,0.026074667771657307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,3,0.026629333694775898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,7,0.027999999622503918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,7,0.0277813325325648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,15,0.03465600063403448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,3,0.027002667387326557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,15,0.034634667138258614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,31,0.03498133271932602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,31,0.035061334570248924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,63,0.03514666606982549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,63,0.0351946676770846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,127,0.011247999966144562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,127,0.007786666974425316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,255,0.01033599985142549
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,255,0.021754667162895203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,511,0.02865600089232127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,511,0.01858666663368543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,1023,0.04599999884764353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,2047,0.04081599911053976
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,1023,0.03276266654332479
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,2047,0.06692799925804138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,4095,0.08559999863306682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,4095,0.05981333553791046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,float16,8191,0.12186132868131001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,1,0.025770666698614757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,1,128,1,float16,fp8,8191,0.08738666772842407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,1,0.02625600000222524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,7,0.027802666028340656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,3,0.026928000152111053
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,3,0.02718399961789449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,7,0.028031999866167705
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,15,0.03475733349720637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,15,0.03460799902677536
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,31,0.034847999612490334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,31,0.035242666800816856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,63,0.03533333291610082
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,63,0.03501333296298981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,127,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,255,0.03031466652949651
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,127,0.007376000285148621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,255,0.015125333021084467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,511,0.04610666632652283
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,1023,0.061610668897628784
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,511,0.02146133283774058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,1023,0.02796799937884013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,2047,0.08443199594815572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,2047,0.04118400067090988
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,4095,0.1204853355884552
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,4095,0.061706667145093284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,1,0.025813333690166473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,float16,8191,0.1930613319079081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,2,128,1,float16,fp8,8191,0.1002293328444163
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,1,0.026314665873845417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,3,0.027263998985290527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,3,0.02699733277161916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,7,0.027818667391935985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,7,0.027930667002995808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,15,0.03482666611671448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,15,0.034832000732421875
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,31,0.03506666670242945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,63,0.03537066777547201
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,31,0.035029334326585136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,63,0.03517866631348928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,127,0.025498665869235992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,127,0.01102399950226148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,255,0.04914666712284088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,511,0.025631998976071674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,255,0.019839999576409657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,511,0.06418133278687795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,1023,0.08086933195590973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,1023,0.03835733234882355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,2047,0.12139733632405598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,2047,0.05770133435726166
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,4095,0.09762133161226909
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,4095,0.19215466578801474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,float16,8191,0.33351465066274005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,4,128,1,float16,fp8,8191,0.17880533138910928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,1,0.026234666506449383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,1,0.026538667579491932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,3,0.026890667776266735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,3,0.027376001079877216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,7,0.027829334139823914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,7,0.028362666567166645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,15,0.034847999612490334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,31,0.03517866631348928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,15,0.0349440003434817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,31,0.03507733345031738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,63,0.03551999976237615
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,63,0.0352960005402565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,127,0.044394666949907936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,255,0.051957334081331887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,127,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,255,0.020975999534130096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,511,0.06892266869544983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,511,0.03323200096686681
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,1023,0.10242133339246114
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,1023,0.05312533179918925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,2047,0.17434666554133096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,2047,0.09318400422732036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,4095,0.3091040054957072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,4095,0.17463467518488565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,float16,8191,0.580074667930603
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,1,0.04744000236193339
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,16,8,128,1,float16,fp8,8191,0.33671998977661133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,1,0.04823466638724009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,3,0.04939199984073639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,3,0.04996266464392344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,7,0.051455999414126076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,7,0.05169600248336792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,15,0.06390933195749919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,15,0.06433600187301636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,31,0.06467199822266896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,31,0.0641546646753947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,63,0.06462400158246358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,63,0.0642986645301183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,127,0.016224000602960587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,127,0.008698666468262672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,255,0.031119999786218006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,255,0.01775466650724411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,511,0.048021331429481506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,1023,0.042303999265034996
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,511,0.031701333820819855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,1023,0.06333866715431213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,float16,2047,0.08521599570910136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,1,128,1,float16,fp8,2047,0.05948266883691152
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,1,0.0473280002673467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,3,0.049914668003718056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,1,0.048437332113583885
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,3,0.049456000328063965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,7,0.05180799961090088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,7,0.051327998439470925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,15,0.06426666676998138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,15,0.06400533517201741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,31,0.06407466530799866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,127,0.026250667870044708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,31,0.0645066648721695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,63,0.0647626668214798
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,63,0.06482666730880737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,127,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,255,0.04947733382383982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,255,0.02162666618824005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,511,0.06519466638565063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,1023,0.0812799980243047
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,511,0.02775999903678894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,1023,0.04200000067551931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,float16,2047,0.12263466914494832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,2,128,1,float16,fp8,2047,0.06108800073464712
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,1,0.04756799836953481
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,1,0.04837333162625631
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,3,0.049365331729253135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,3,0.05018133421738943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,7,0.05136000116666158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,7,0.05186666548252106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,15,0.06436799963315327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,15,0.06419733166694641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,31,0.06483200192451477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,63,0.06487466891606648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,31,0.06435200075308482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,63,0.06459733347098033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,127,0.04513066510359446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,127,0.016282666474580765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,255,0.052842666705449425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,255,0.02182399978240331
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,511,0.03358400116364161
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,511,0.06934399902820587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,1023,0.10372266173362732
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,1023,0.0536053329706192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,float16,2047,0.17680533727010092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,4,128,1,float16,fp8,2047,0.09413333733876546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,1,0.04781866570313772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,1,0.04862933357556661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,3,0.049551998575528465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,3,0.05062933266162872
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,7,0.052149335543314614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,7,0.05157866577307383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,15,0.06448533137639363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,31,0.06468266745408376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,31,0.06485866506894429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,63,0.06740800042947133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,63,0.06518400212128957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,15,0.06485333542029063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,127,0.08150933186213176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,127,0.028602667152881622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,255,0.09326933821042378
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,511,0.06037333110968272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,255,0.04058133314053217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,511,0.12493866682052612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,1023,0.19307732582092285
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,1023,0.10094400246938069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,float16,2047,0.33112533887227374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,16,8,128,1,float16,fp8,2047,0.18086934089660645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,1,0.09170132875442505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,1,0.09010666608810425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,3,0.09310400485992432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,3,0.09462933739026387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,15,0.12243733803431193
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,7,0.0971999963124593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,7,0.09815999865531921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,31,0.12274133165677388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,15,0.12235732873280843
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,63,0.12336533268292744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,31,0.12213333447774251
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,63,0.12296000123023987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,127,0.02736533433198929
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,127,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,255,0.052416001756985985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,255,0.03146133323510488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,511,0.06689066688219707
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,1,0.09005332986513774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,511,0.04171200096607208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,1,0.09177600344022115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,fp8,1023,0.05978133281071981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,1,128,1,float16,float16,1023,0.08334400256474812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,3,0.09336533149083455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,3,0.09468266367912292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,7,0.09737599889437358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,7,0.09815466403961182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,15,0.12231466174125671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,15,0.1225440005461375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,31,0.12299199899037679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,31,0.1223466694355011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,63,0.12309333682060242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,63,0.12362666924794515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,127,0.046709333856900535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,127,0.017690667261679966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,255,0.054085334142049156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,255,0.024800000091393787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,511,0.07107200225194295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,511,0.03802666564782461
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,float16,1023,0.10546132922172546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,1,0.09196266531944275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,1,0.09038933118184407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,2,128,1,float16,fp8,1023,0.056874667604764305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,3,0.0935093363126119
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,3,0.09505599737167358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,7,0.0974826713403066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,15,0.12250666817029317
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,7,0.09884267052014668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,15,0.12273599704106648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,31,0.12338133653004964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,31,0.1228053371111552
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,63,0.12903466820716858
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,63,0.12337600191434224
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,127,0.02917333443959554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,127,0.0830080012480418
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,255,0.04148799926042557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,255,0.09510933359464009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,511,0.12743467092514038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,511,0.061535999178886414
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,float16,1023,0.19477333625157675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,4,128,1,float16,fp8,1023,0.1020960013071696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,1,0.0906986693541209
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,3,0.09387200077374776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,1,0.09282666444778442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,3,0.09568533301353455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,7,0.09789866209030151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,7,0.0997920036315918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,15,0.12334932883580525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,31,0.1234773298104604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,15,0.12320533394813538
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,31,0.12918399771054587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,63,0.1322719951470693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,63,0.1295840044816335
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,127,0.15199466546376547
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,127,0.05524266759554545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,255,0.17473065853118896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,255,0.073594664533933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,1,0.009514666472872099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,511,0.2360639969507853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,511,0.11197866996129353
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,float16,1023,0.3689546585083008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,16,8,128,1,float16,fp8,1023,0.1892426609992981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,1,0.00980266680320104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,7,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,3,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,3,0.00973866693675518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,15,0.009733333562811216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,7,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,31,0.01051733394463857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,15,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,31,0.0106133334338665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,63,0.0122079998254776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,63,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,127,0.012213333199421564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,255,0.01251199965675672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,255,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,1023,0.013429333766301474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,511,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,511,0.028389332195123036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,1023,0.0102186668664217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,2047,0.017397332936525345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,2047,0.011498666057984034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,4095,0.018346666047970455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,4095,0.01331199953953425
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,8191,0.027232001225153606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,16383,0.03335466732581457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,8191,0.018565333137909572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,16383,0.0227360005180041
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,32767,0.05202133456865946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,32767,0.04126933217048645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,65535,0.07711466650168101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,1,0.009594666461149851
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,65535,0.05622399846712748
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,1,0.009743999689817429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,float16,131071,0.12882133324941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,3,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,1,128,1,float16,fp8,131071,0.0979306697845459
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,3,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,7,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,7,0.009839999799927076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,15,0.009877333417534828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,15,0.00984533317387104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,31,0.010666667173306147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,31,0.010384000216921171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,63,0.012448000411192576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,127,0.012357333054145178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,127,0.012304000556468964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,63,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,511,0.012821332861979803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,255,0.012058666596810022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,255,0.01268799975514412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,511,0.007930666829148928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,2047,0.02181333303451538
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,1023,0.008650666723648706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,1023,0.013301332791646322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,2047,0.010677333921194077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,4095,0.012842666357755661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,4095,0.025301332275072735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,8191,0.03621333340803782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,8191,0.017829333742459614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,16383,0.05323733389377594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,16383,0.028981332977612812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,32767,0.08052266637484233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,32767,0.04148799926042557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,65535,0.12615999579429626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,65535,0.06994133194287618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,fp8,131071,0.11460799972216289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,2,128,1,float16,float16,131071,0.21089067061742148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,1,0.00961599995692571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,1,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,3,0.00985599992175897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,3,0.009722666814923286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,7,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,7,0.009775999933481216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,15,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,31,0.010608000059922537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,15,0.010144000252087912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,31,0.010687999427318573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,63,0.012240000069141388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,127,0.012293333808581034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,63,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,127,0.012341332932313284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,255,0.0124746672809124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,255,0.007376000285148621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,511,0.012965332716703415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,511,0.007733333234985669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,1023,0.01921066641807556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,1023,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,2047,0.011402666568756104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,2047,0.02869333326816559
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,4095,0.040362666050593056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,4095,0.01886933296918869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,8191,0.05817066629727682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,8191,0.024442667762438457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,16383,0.08593066533406575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,16383,0.042303999265034996
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,32767,0.1232319970925649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,32767,0.06313600142796834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,65535,0.1999839941660563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,65535,0.10761599739392598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,1,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,3,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,1,0.009759999811649323
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,float16,131071,0.3568106492360433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,4,128,1,float16,fp8,131071,0.191594660282135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,3,0.009999999776482582
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,7,0.010058666889866194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,15,0.010165333126982054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,7,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,15,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,31,0.010458666831254959
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,31,0.010816000401973724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,63,0.012362666428089142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,63,0.012346666306257248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,127,0.006586666529377301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,511,0.018992000569899876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,255,0.01341333364446958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,255,0.00744000015159448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,127,0.010362666721145311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,511,0.008570666735370954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,1023,0.027962667246659596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,1023,0.014111999422311783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,2047,0.042538667718569435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,2047,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,4095,0.06677333513895671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,4095,0.026613332331180573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,8191,0.08442667126655579
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,8191,0.039488000174363456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,16383,0.11934933066368103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,16383,0.0584853341182073
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,32767,0.09896533687909444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,32767,0.1895786722501119
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,65535,0.3291146755218506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,65535,0.1804479956626892
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,1,0.17479999860127768
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,fp8,131071,0.3434240023295085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,16,8,128,1,float16,float16,131071,0.6154506603876749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,3,0.18035733699798584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,1,0.17822933197021484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,3,0.18268799781799316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,7,0.18995199600855509
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,7,0.19183466831843057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,15,0.2389226754506429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,15,0.23941334088643393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,31,0.24007999897003174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,31,0.23896533250808716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,63,0.24127467473347983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,63,0.24002132813135782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,127,0.04844266672929128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,127,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,255,0.05630933245023092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,255,0.03736000011364619
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,1,0.1745599905649821
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,1,0.17824532588322958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,3,0.18056533734003702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,fp8,511,0.05505066613356272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,1,128,1,float16,float16,511,0.07348266740640004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,3,0.18291733662287393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,7,0.18979734182357788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,7,0.19249600172042847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,15,0.2391093373298645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,31,0.24073066314061484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,15,0.23953600724538168
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,31,0.23921600977579752
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,63,0.2505120038986206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,63,0.24025599161783853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,127,0.0860640009244283
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,127,0.03338133295377096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,255,0.0473280002673467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,255,0.0974666674931844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,float16,511,0.12984533111254373
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,1,0.1750026742617289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,2,128,1,float16,fp8,511,0.0666133314371109
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,1,0.17854932943979898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,3,0.18092799186706543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,3,0.18382932742436728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,7,0.1901599963506063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,7,0.19313067197799683
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,15,0.2400266726811727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,15,0.24038932720820108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,31,0.2505813241004944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,31,0.24079465866088867
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,63,0.25301865736643475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,127,0.15450132886568704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,63,0.25255467494328815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,127,0.05748266478379568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,255,0.1775253415107727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,255,0.07561066746711731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,float16,511,0.23989866177241007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,4,128,1,float16,fp8,511,0.11374933520952861
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,1,0.17554134130477905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,1,0.1794453263282776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,3,0.18183465798695883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,3,0.18546134233474731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,7,0.1946720083554586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,7,0.1922666629155477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,15,0.24864532550175986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,15,0.24235200881958008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,31,0.2525599996248881
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,31,0.25487999121348065
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,63,0.25704532861709595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,63,0.2567360003789266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,127,0.29472533861796063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,127,0.10371733705202739
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,255,0.1394719978173574
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,255,0.3391253153483073
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,float16,511,0.4581173261006673
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,16,8,128,1,float16,fp8,511,0.2143253286679586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,1,0.3449920018513997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,1,0.35149868329366046
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,3,0.360858678817749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,3,0.3551146586736043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,7,0.3787093162536621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,7,0.37433067957560223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,15,0.4726773500442505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,15,0.47358934084574383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,31,0.4796266555786133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,31,0.47339733441670734
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,63,0.4904213349024455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,63,0.4800479809443156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,127,0.09004799524943034
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,127,0.06366399923960368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,1,0.34479467074076336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,float16,255,0.10247466961542766
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,1,0.3511146704355876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,1,128,1,float16,fp8,255,0.07764266431331635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,3,0.3553973436355591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,3,0.36114664872487384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,7,0.375055988629659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,7,0.3795359929402669
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,15,0.4745440085728963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,15,0.478000005086263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,31,0.4893973271052043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,31,0.4806613524754842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,63,0.4930613438288371
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,63,0.49342934290568036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,127,0.16107733050982156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,127,0.06540266672770183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,1,0.3449866771697998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,3,0.35814400513966876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,1,0.35157867272694904
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,3,0.36203734079996747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,float16,255,0.1839253306388855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,2,128,1,float16,fp8,255,0.08461333314577739
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,7,0.3874826828638713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,7,0.38362665971120197
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,15,0.48636265595753986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,15,0.4837119976679484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,31,0.4935946861902873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,63,0.49778131643931073
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,31,0.49453334013621014
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,63,0.4986666838328044
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,127,0.29806933800379437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,127,0.10729066530863444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,1,0.35372265179951984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,1,0.34962666034698486
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,float16,255,0.34486401081085205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,3,0.3750400145848592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,4,128,1,float16,fp8,255,0.14365333318710327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,3,0.37009068330128986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,7,0.4018186728159587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,7,0.4028373161951701
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,15,0.49167466163635254
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,15,0.4975680112838745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,31,0.5022826592127482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,31,0.5023946762084961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,63,0.5061546564102173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,63,0.5066933234532675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,127,0.576314647992452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,1,0.009152000149091085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,3,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,1,0.009061333412925402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,7,0.009141333401203156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,3,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,127,0.20082134008407593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,31,0.011530666301647821
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,float16,255,0.6666400035222372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,15,0.009919999788204828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,7,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,31,0.011839999506870905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,15,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,63,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,16,8,128,1,float16,fp8,255,0.27217066287994385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,63,0.011706666400035223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,127,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,255,0.013514666507641474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,127,0.011781333635250727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,255,0.013765333841244379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,511,0.009370666618148485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,511,0.012858666479587555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,1023,0.013701333353916803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,1023,0.010437333335479101
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,2047,0.022229333718617756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,2047,0.012965332716703415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,4095,0.015439999600251516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,4095,0.02554133286078771
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,8191,0.03664533297220866
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,8191,0.026842666169007618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,16383,0.053727999329566956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,16383,0.03398933261632919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,32767,0.08177599807580312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,32767,0.06022400160630544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,1,0.00892800030608972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,1,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,3,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,float16,65535,0.1288266678651174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,3,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,1,128,1,float16,fp8,65535,0.0950933297475179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,7,0.009050666665037474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,7,0.009418666362762451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,15,0.009701333319147428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,15,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,63,0.01157333329319954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,31,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,31,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,63,0.011829332758982977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,127,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,511,0.01313599944114685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,255,0.012629333883523941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,255,0.007813333223263422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,511,0.008346666892369589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,127,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,1023,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,1023,0.01878400022784869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,2047,0.02887466549873352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,2047,0.012479999413092932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,4095,0.040448000033696495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,8191,0.05877333382765452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,4095,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,16383,0.08713600039482117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,8191,0.025797332326571148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,16383,0.04497600098450979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,32767,0.12329600254694621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,32767,0.06559999783833821
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,1,0.009226666763424873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,1,0.009039999917149544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,3,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,float16,65535,0.19998933871587118
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,3,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,2,128,1,float16,fp8,65535,0.1099626620610555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,7,0.009082666908701261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,7,0.009296000003814697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,15,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,31,0.011589333415031433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,63,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,31,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,15,0.010037333394090334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,63,0.012047999848922094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,127,0.010106666634480158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,255,0.013568000247081121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,127,0.006501333167155583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,255,0.007536000261704127
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,511,0.019066666563351948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,511,0.00860799973209699
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,1023,0.014106666048367819
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,1023,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,2047,0.01762666677435239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,4095,0.06700799862543742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,2047,0.04278400043646494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,4095,0.026799999177455902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,8191,0.08461333314577739
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,16383,0.059024001161257424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,16383,0.11961600184440613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,8191,0.03962666789690653
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,32767,0.09909866253534953
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,32767,0.1914666692415873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,float16,65535,0.33241599798202515
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,1,0.008943999807039896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,4,128,1,float16,fp8,65535,0.18078400691350302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,7,0.00903466654320558
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,7,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,3,0.009258666386206945
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,1,0.009125333279371262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,3,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,15,0.009946666657924652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,15,0.0100426667680343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,63,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,31,0.011610666910807291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,31,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,63,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,127,0.010656000425418219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,255,0.02062400057911873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,127,0.006517333288987477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,255,0.008282666405042013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,511,0.02790933350721995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,511,0.013562666873137156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,1023,0.04539733131726583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,1023,0.020015999674797058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,2047,0.06507733464241028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,2047,0.025610665480295818
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,4095,0.08363733688990276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,8191,0.11853333314259847
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,4095,0.038005332152048744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,8191,0.059978668888409935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,16383,0.0983786682287852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,16383,0.18863999843597412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,32767,0.32816000779469806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,32767,0.17926400899887085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,1,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,1,0.00891733355820179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,float16,65535,0.6061013142267863
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,3,0.009178666397929192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,3,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,16,8,128,1,float16,fp8,65535,0.3415893316268921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,7,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,7,0.009152000149091085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,15,0.0100426667680343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,15,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,31,0.012015999605258306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,63,0.011621333658695221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,63,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,31,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,127,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,127,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,255,0.012522666404644648
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,255,0.007813333223263422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,511,0.013343999783198038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,511,0.00821333316465219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,1023,0.01893866683046023
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,2047,0.02918400118748347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,1023,0.00997866690158844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,2047,0.01250133290886879
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,4095,0.04141333450873693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,4095,0.01958400011062622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,8191,0.026309333741664886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,8191,0.05952000121275584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,16383,0.08825600147247314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,16383,0.044906665881474815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,32767,0.12462932864824931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,1,0.008965333302815756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,32767,0.06554666658242543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,1,0.009173333023985228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,3,0.01002133327225844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,3,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,fp8,65535,0.10982400178909302
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,1,128,1,float16,float16,65535,0.20053333044052124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,7,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,7,0.009226666763424873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,15,0.010133333504199982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,15,0.010149333626031876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,31,0.011567999919255575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,31,0.011695999652147293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,63,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,63,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,127,0.006490666419267654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,127,0.010191999996701876
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,255,0.013728000223636627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,511,0.00855466661353906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,511,0.019120000302791595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,255,0.007429333403706551
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,2047,0.04307733476161957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,1023,0.027621333797772724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,1023,0.014101333916187286
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,2047,0.01743999992807706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,4095,0.06672533353169759
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,4095,0.026704000929991405
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,8191,0.0846506655216217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,16383,0.12051733334859212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,8191,0.03959999978542328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,16383,0.05851200222969055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,32767,0.19085333744684854
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,32767,0.09934932986895244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,1,0.009248000259200731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,fp8,65535,0.18040533860524496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,2,128,1,float16,float16,65535,0.33288000027338666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,3,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,3,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,7,0.009130666653315226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,7,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,15,0.009861333295702934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,15,0.010010666524370512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,63,0.011770666887362799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,31,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,31,0.011589333415031433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,63,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,127,0.006591999903321266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,127,0.010693332801262537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,255,0.008223999912540117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,255,0.02049066623051961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,511,0.028042666614055634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,511,0.013370666652917862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,1023,0.04552533229192098
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,1023,0.020037333170572918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,2047,0.06524266799290974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,2047,0.025498665869235992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,4095,0.08331733445326488
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,4095,0.03825599948565165
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,8191,0.11936533451080322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,8191,0.06084799766540527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,16383,0.09821866949399312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,16383,0.19024533033370972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,32767,0.3298880060513814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,32767,0.17934399843215942
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,1,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,1,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,float16,65535,0.611135999361674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,8,4,128,1,float16,fp8,65535,0.3417653242746989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,3,0.009045333291093508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,3,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,7,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,7,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,15,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,15,0.009328000247478485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,31,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,63,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,31,0.0100853331387043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,63,0.011653333902359009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,127,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,127,0.011685332904259363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,255,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,511,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,1023,0.017221332838137943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,255,0.01163200040658315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,511,0.016127999871969223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,1023,0.01730666682124138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,2047,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,2047,0.0174346665541331
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,4095,0.017808000246683758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,8191,0.01817600056529045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,4095,0.017450666675964992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,16383,0.01956266661485036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,8191,0.012671999633312225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,16383,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,32767,0.02382933348417282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,32767,0.01911466692884763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,65535,0.02378133436044057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,65535,0.02899733434120814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,1,0.008901333436369896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,1,0.010106666634480158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,float16,131071,0.039034667114416756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,1,128,1,float16,fp8,131071,0.031712000568707786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,3,0.008879999940594038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,3,0.009205333267649015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,7,0.008997333546479544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,7,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,31,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,15,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,15,0.00922133338948091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,31,0.010112000008424124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,63,0.011616000284751257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,127,0.0116799995303154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,63,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,127,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,255,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,255,0.011600000162919363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,511,0.01621866722901662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,1023,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,511,0.016250666230916977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,1023,0.017210666090250015
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,2047,0.01748266691962878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,2047,0.01754133279124896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,4095,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,4095,0.01020800011853377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,8191,0.018618666877349217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,8191,0.012432000289360682
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,16383,0.02011200040578842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,32767,0.01882133384545644
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,32767,0.024661332368850708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,16383,0.01470400020480156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,65535,0.03046400099992752
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,65535,0.02380799998839696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,1,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,float16,131071,0.05649599929650625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,2,128,1,float16,fp8,131071,0.035642666121323906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,1,0.009482666850090027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,3,0.00897066667675972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,7,0.008863999818762144
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,3,0.009301333377758661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,7,0.009578666960199675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,15,0.009338666374484697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,63,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,31,0.010064000263810158
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,63,0.011717333147923151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,15,0.009472000102202097
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,127,0.011999999483426413
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,31,0.01007466639081637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,127,0.011621333658695221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,255,0.011584000041087469
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,511,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,511,0.01624533285697301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,255,0.012080000092585882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,2047,0.014794666320085526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,1023,0.017344000438849132
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,1023,0.017184000462293625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,2047,0.00895999992887179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,4095,0.015658666690190632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,4095,0.010234666367371878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,8191,0.01952533299724261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,8191,0.012335999558369318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,16383,0.021242665747801464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,16383,0.014890667051076889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,32767,0.034815999368826546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,32767,0.02162133405605952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,65535,0.046800002455711365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,65535,0.028330666323502857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,float16,131071,0.07689066727956136
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,1,0.009136000027259191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,3,0.009130666653315226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,3,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,1,0.00938666673998038
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,8,4,128,1,float16,fp8,131071,0.04234133164087931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,7,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,7,0.009317333499590555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,15,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,31,0.009952000031868616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,15,0.009535999968647957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,31,0.01027199998497963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,63,0.011690666278203329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,63,0.011802667131026586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,127,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,127,0.01163200040658315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,255,0.0116799995303154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,255,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,511,0.016352000335852306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,511,0.01637866720557213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,1023,0.017504000415404636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,2047,0.017530667285124462
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,1023,0.01746133342385292
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,4095,0.015962666521469753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,2047,0.01764800027012825
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,4095,0.010682666053374609
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,8191,0.019237333287795384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,8191,0.012671999633312225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,16383,0.0206133338312308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,16383,0.015029333531856537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,32767,0.01921066641807556
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,32767,0.025221332907676697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,65535,0.031167998909950256
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,65535,0.024160000185171764
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,1,0.009136000027259191
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,float16,131071,0.05861333509286245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,1,128,1,float16,fp8,131071,0.036277333895365395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,1,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,3,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,3,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,7,0.009359999870260557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,7,0.00901333304742972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,15,0.00926399976015091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,31,0.0102186668664217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,31,0.010026666646202406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,63,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,15,0.009568000212311745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,127,0.011968000481526056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,127,0.011727999895811081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,63,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,511,0.01628799984852473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,511,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,255,0.011776000261306763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,1023,0.01741333305835724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,255,0.012042666474978128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,1023,0.017514667163292568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,2047,0.01563199982047081
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,4095,0.010298666854699453
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,4095,0.01629866659641266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,2047,0.008949333180983862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,8191,0.02013333390156428
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,8191,0.01251199965675672
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,16383,0.021957332889238994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,16383,0.015050667027632395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,32767,0.02205866575241089
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,32767,0.0355679988861084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,65535,0.04747733473777771
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,65535,0.028378665447235107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,float16,131071,0.07784000039100647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,1,0.009072000160813332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,2,128,1,float16,fp8,131071,0.04204800228277842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,3,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,1,0.009530666594703993
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,7,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,3,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,7,0.009397333487868309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,15,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,15,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,31,0.01002133327225844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,31,0.010213333492477735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,63,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,63,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,127,0.011802667131026586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,255,0.01181866725285848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,255,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,511,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,511,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,127,0.01173866664369901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,1023,0.013477332890033722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,1023,0.008037333066264788
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,2047,0.01571200042963028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,2047,0.009098666409651438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,4095,0.016607999801635742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,4095,0.0103946669648091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,8191,0.02605866640806198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,8191,0.014303999642531076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,16383,0.031167998909950256
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,16383,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,32767,0.04952533543109894
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,32767,0.024304000039895374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,65535,0.07392533123493195
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,65535,0.04670399924119314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,1,0.015082667271296183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,1,0.015168000012636185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,float16,131071,0.12576533357302347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,3,0.015146666516860327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,8,4,128,1,float16,fp8,131071,0.0670773337284724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,3,0.01543466622630755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,7,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,7,0.015290666371583939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,15,0.016415999581416447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,15,0.01655999943614006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,31,0.019802667200565338
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,31,0.019962667177120846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,63,0.020175999651352566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,255,0.008122666428486506
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,63,0.019845332950353622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,127,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,255,0.013621332744757334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,127,0.006655999769767125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,511,0.00914666677514712
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,511,0.0191040001809597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,1023,0.027989332874615986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,1023,0.015130666395028433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,2047,0.01858666663368543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,2047,0.04326933125654856
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,4095,0.06764266888300578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,4095,0.028789333999156952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,8191,0.08630933364232381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,8191,0.042266666889190674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,16383,0.1218773325284322
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,16383,0.06156266729036967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,1,0.014981333166360855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,3,0.015024000157912573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,3,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,float16,32767,0.19401599963506064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,1,128,1,float16,fp8,32767,0.10136000315348308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,7,0.015344000111023584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,7,0.01562133307258288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,15,0.016528000434239704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,15,0.016682667036851246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,31,0.019871999820073444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,31,0.019973333925008774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,63,0.020058666666348774
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,63,0.020080000162124634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,127,0.01080000028014183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,127,0.006709333509206772
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,255,0.020666666328907013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,511,0.02788266787926356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,255,0.008314666648705801
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,1023,0.045237332582473755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,511,0.013455999394257864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,1023,0.020186666399240494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,2047,0.025813333690166473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,2047,0.06550399959087372
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,4095,0.03847466657559077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,4095,0.08361066381136577
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,8191,0.11964799960454305
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,8191,0.06058666606744131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,16383,0.19126399358113608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,1,0.015002666662136713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,16383,0.09844799836476643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,1,0.01533866673707962
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,float16,32767,0.33265066146850586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,3,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,2,128,1,float16,fp8,32767,0.17961066961288452
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,3,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,7,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,7,0.01545599972208341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,15,0.016522667060295742
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,15,0.016970666746298473
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,31,0.019765333582957584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,31,0.020117333779732387
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,63,0.01998399943113327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,63,0.02012266715367635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,127,0.015626666446526844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,127,0.0069973332186539965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,255,0.029845332105954487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,255,0.013471999516089758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,511,0.04602666695912679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,511,0.01964266722400983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,1023,0.025008000433444977
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,1023,0.06137600044409434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,2047,0.08304533362388611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,2047,0.038047999143600464
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,4095,0.11947199702262878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,8191,0.19104532400767008
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,4095,0.05885333319505056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,8191,0.0974133312702179
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,16383,0.33085866769154865
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,16383,0.1787359913190206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,3,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,float16,32767,0.6109493176142374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,1,0.009242666885256767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,1,0.009503999724984169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,3,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,8,4,128,1,float16,fp8,32767,0.34142398834228516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,7,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,7,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,15,0.009450666606426239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,15,0.009642666826645533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,31,0.010159999753038088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,31,0.010378666842977205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,63,0.011957333733638128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,63,0.011663999408483505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,127,0.012005332857370377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,127,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,255,0.012330666184425354
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,511,0.01655999943614006
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,511,0.016730666160583496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,255,0.011893333246310553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,1023,0.017759999881188076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,1023,0.017658667018016178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,2047,0.01714666684468587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,2047,0.009578666960199675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,4095,0.010751999914646149
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,4095,0.017466666797796886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,8191,0.020986666282018025
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,16383,0.015317333241303762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,8191,0.012863999853531519
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,16383,0.023242667317390442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,32767,0.03608533243338267
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,32767,0.02295999974012375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,65535,0.04977599779764811
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,65535,0.029274667302767437
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,1,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,1,0.00955200009047985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,3,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,float16,131071,0.07959466675917308
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,3,0.009642666826645533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,1,128,1,float16,fp8,131071,0.04241600135962168
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,7,0.009301333377758661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,7,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,15,0.009370666618148485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,15,0.009797333429257074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,31,0.010175999874869982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,31,0.010490667074918747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,127,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,63,0.01180800050497055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,63,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,127,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,255,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,255,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,511,0.01653333380818367
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,1023,0.013450667262077332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,511,0.016602666427691776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,1023,0.008101333553592363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,2047,0.017055999487638474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,2047,0.008954666554927826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,4095,0.017952000101407368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,4095,0.010421333213647207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,8191,0.02722666660944621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,8191,0.014208000153303146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,16383,0.017423999806245167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,16383,0.03230933348337809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,32767,0.025557334224383037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,32767,0.05171733101209005
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,1,0.009226666763424873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,65535,0.04613333443800608
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,1,0.009610666582981745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,65535,0.07502399881680806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,3,0.009290666629870733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,float16,131071,0.12703999876976013
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,3,0.009685333197315535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,2,128,1,float16,fp8,131071,0.06666133304437001
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,7,0.009178666397929192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,7,0.009541333342591921
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,31,0.01022933361430963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,15,0.009509333098928133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,15,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,31,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,63,0.011898666620254517
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,63,0.011850666254758835
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,127,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,127,0.011989332735538483
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,255,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,1023,0.013167999684810638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,255,0.012202666451533636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,511,0.007631999750932057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,511,0.01259200026591619
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,2047,0.021669333179791767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,1023,0.008117333054542542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,2047,0.010175999874869982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,4095,0.025040000677108765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,4095,0.011882666498422623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,8191,0.03608533243338267
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,8191,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,16383,0.054144000013669334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,16383,0.028058665494124096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,32767,0.08120533327261607
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,32767,0.03919466584920883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,65535,0.1264693339665731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,1,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,1,0.014805333067973455
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,65535,0.06710933148860931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,3,0.015087999403476715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,float16,131071,0.21051200230916342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,8,4,128,1,float16,fp8,131071,0.11206400394439697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,7,0.015696000307798386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,3,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,7,0.015775999675194424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,15,0.01933866615096728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,15,0.019215999792019527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,31,0.019487999379634857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,31,0.019519999623298645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,63,0.01978133370478948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,63,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,127,0.007226666435599327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,127,0.010949333508809408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,255,0.021151999632517498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,255,0.008943999807039896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,511,0.028207999964555103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,511,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,1023,0.045423999428749084
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,1023,0.021642667551835377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,2047,0.06573333342870076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,2047,0.02739733209212621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,4095,0.08468266328175862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,4095,0.041082667807737984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,8191,0.12129599849383037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,1,0.014773332824309668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,8191,0.0629120022058487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,1,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,float16,16383,0.19335466623306274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,1,128,1,float16,fp8,16383,0.10072533289591472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,3,0.015178666760524115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,3,0.015413332730531693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,7,0.01571200042963028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,7,0.015861333658297855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,15,0.019551999866962433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,15,0.0195573332409064
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,127,0.007530666887760162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,31,0.019600000232458115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,31,0.019365333020687103
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,127,0.015509333461523056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,63,0.019541333119074505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,63,0.01960533360640208
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,255,0.02976000060637792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,511,0.045850664377212524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,255,0.013264000415802002
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,511,0.020090666910012562
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,1023,0.025381334125995636
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,1023,0.06128533184528351
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,2047,0.03857066730658213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,2047,0.08377599716186523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,4095,0.11995733777681987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,4095,0.057962665955225624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,1,0.015013333410024643
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,8191,0.19225066900253296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,8191,0.09803199768066406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,1,0.015024000157912573
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,3,0.01540800059835116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,float16,16383,0.33485865592956543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,3,0.015647999942302704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,7,0.01571200042963028
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,2,128,1,float16,fp8,16383,0.1791093349456787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,7,0.016186666985352833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,15,0.01966399947802226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,15,0.01951466624935468
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,31,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,63,0.019653332730134327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,63,0.019578666736682255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,31,0.019567999988794327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,127,0.025285333395004272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,127,0.011029332876205444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,255,0.04900800188382467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,255,0.019621333728233974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,511,0.0641546646753947
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,1023,0.08051200211048126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,511,0.025093334416548412
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,1023,0.03812266637881597
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,2047,0.1206773320833842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,2047,0.057146668434143066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,4095,0.19165867567062378
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,4095,0.09732799728711446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,8191,0.1784906586011251
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,8191,0.33297065893809
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,1,0.025797332326571148
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,1,0.026320000489552815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,float16,16383,0.6137439807256063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,3,0.026954665780067444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,3,0.02722666660944621
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,8,4,128,1,float16,fp8,16383,0.34059735139211017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,7,0.02770666778087616
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,7,0.027818667391935985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,15,0.034815999368826546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,15,0.03461866577466329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,31,0.03505599995454153
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,31,0.0349386657277743
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,63,0.03495466709136963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,63,0.03535466641187668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,127,0.007365333537260692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,127,0.01623999948302905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,255,0.030224000414212544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,255,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,511,0.021738665799299877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,511,0.04637333254019419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,1023,0.061706667145093284
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,1023,0.027429332335789997
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,2047,0.08462933699289958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,2047,0.041290665666262306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,3,0.02677333354949951
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,float16,4095,0.1218239963054657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,1,0.026357332865397137
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,1,0.02571200082699458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,7,0.027786667148272198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,3,0.027130665878454845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,1,128,1,float16,fp8,4095,0.061333333452542625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,7,0.02802666773398717
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,15,0.03496533383925756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,31,0.034917332231998444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,31,0.035360001027584076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,15,0.034874667723973594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,63,0.03532800078392029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,63,0.035216001172860466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,127,0.025301332275072735
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,127,0.011007999380429586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,255,0.019920000185569126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,255,0.04934933284918467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,1023,0.08045866588751475
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,511,0.06439466774463654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,511,0.025461333493391674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,1023,0.03878399978081385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,2047,0.12231999635696411
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,2047,0.058176000912984215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,1,0.025920001169045765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,1,0.026426665484905243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,3,0.02681066592534383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,float16,4095,0.1926506757736206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,2,128,1,float16,fp8,4095,0.09830400347709656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,3,0.0273333340883255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,7,0.02779199928045273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,7,0.028384000062942505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,15,0.03482666611671448
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,15,0.03491200009981791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,31,0.03521066655715307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,31,0.03495466709136963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,63,0.035375999907652535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,63,0.03517866631348928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,127,0.04437866806983948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,127,0.016117333124081295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,255,0.05221333106358846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,255,0.021066665649414062
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,511,0.03340800106525421
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,511,0.06874666611353557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,1023,0.10318932930628459
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,1023,0.05339199801286062
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,2047,0.17622933785120645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,2047,0.0939626693725586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,float16,4095,0.31407467524210614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,1,0.04762133459250132
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,1,0.04841066896915436
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,3,0.04996266464392344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,8,4,128,1,float16,fp8,4095,0.17484800020853677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,3,0.04934933284918467
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,7,0.051258668303489685
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,7,0.05169600248336792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,15,0.06427200138568878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,15,0.06412800153096516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,31,0.06438399851322174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,31,0.06412266691525777
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,63,0.06471466521422069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,63,0.06473066906134288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,127,0.02621866762638092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,127,0.012357333054145178
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,255,0.049471999208132424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,255,0.02180800090233485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,511,0.027893332143624622
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,511,0.06548266609509786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,1023,0.08167466521263123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,1023,0.041663999358812966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,1,0.04752533137798309
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,1,0.048432002464930214
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,float16,2047,0.12349866827329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,1,128,1,float16,fp8,2047,0.062368000547091164
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,3,0.04947733382383982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,3,0.05009600023428599
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,7,0.05163733164469401
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,15,0.06445333361625671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,7,0.051685333251953125
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,15,0.06423466900984447
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,31,0.06480533381303151
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,31,0.06437333424886067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,63,0.06502399841944377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,63,0.06464533507823944
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,127,0.016202667107184727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,127,0.04515199859937032
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,255,0.02203733225663503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,255,0.05248000224431356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,511,0.06943466762701671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,511,0.03381866713364919
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,1023,0.1032426655292511
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,1023,0.05385600030422211
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,1,0.0476800004641215
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,1,0.048911998669306435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,float16,2047,0.17603200674057007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,2,128,1,float16,fp8,2047,0.09486933549245198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,3,0.04961599906285604
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,3,0.050842667619387306
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,7,0.051557332277297974
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,7,0.05194133520126343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,15,0.06459733347098033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,15,0.06469866633415222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,31,0.06486399968465169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,31,0.06496533254782359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,63,0.06679466863473256
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,63,0.06520533561706543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,127,0.08213333288828532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,127,0.028543998797734577
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,255,0.09427199761072795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,255,0.04036800066630045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,511,0.12610133488972983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,511,0.06052800019582113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,1023,0.19374932845433554
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,1023,0.10107200344403584
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,float16,2047,0.3331039945284526
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,3,0.009349333122372627
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,1,0.009285333255926767
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,1,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,8,4,128,1,float16,fp8,2047,0.18105065822601318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,3,0.009882666791478792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,7,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,7,0.009626666704813639
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,15,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,15,0.009861333295702934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,31,0.010245333115259806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,31,0.010368000095089277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,63,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,63,0.01209066684047381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,127,0.012245333443085352
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,255,0.012159999459981918
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,127,0.012085333466529846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,255,0.01202133297920227
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,511,0.016773333152135212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,511,0.016741332908471424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,1023,0.013487999637921652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,1023,0.008378666515151659
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,2047,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,2047,0.017157333592573803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,4095,0.01829333355029424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,4095,0.010911999891201654
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,8191,0.02717333287000656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,8191,0.014981333166360855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,32767,0.051813334226608276
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,16383,0.03187733391920725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,16383,0.018298666924238205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,32767,0.025973332424958546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,65535,0.04969066878159841
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,65535,0.07634666562080383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,1,0.009408000235756239
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,1,0.009706666693091393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,3,0.009797333429257074
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,7,0.009477333476146063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,3,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,fp8,131071,0.06916800141334534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,7,0.009818666925032934
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,1,128,1,float16,float16,131071,0.12685867150624594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,15,0.009525333220760027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,15,0.009888000165422758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,31,0.010426666587591171
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,31,0.01027199998497963
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,63,0.012165332833925882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,63,0.012069333344697952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,127,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,127,0.012144000579913458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,255,0.012138667205969492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,255,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,511,0.012901333471139273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,511,0.007610666876037915
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,1023,0.013370666652917862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,1023,0.008133333176374435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,2047,0.02189333240191142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,4095,0.024821333587169647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,4095,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,2047,0.010101333260536194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,8191,0.016074666132529575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,8191,0.035786665976047516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,16383,0.053210665782292686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,16383,0.027930667002995808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,32767,0.03938666731119156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,32767,0.08006933331489563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,65535,0.12599466244379678
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,65535,0.0669706662495931
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,1,0.009519999846816063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,3,0.009402666861812273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,1,0.009663999701539675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,3,0.009813333551088968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,fp8,131071,0.11214400331179301
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,2,128,1,float16,float16,131071,0.20895467201868692
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,7,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,7,0.009679999823371569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,15,0.009829333052039146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,15,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,63,0.012138667205969492
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,127,0.012229333321253458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,31,0.0102613332370917
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,63,0.012026666353146235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,127,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,255,0.012746666868527731
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,255,0.00879466657837232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,511,0.012874666601419449
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,511,0.008869333192706108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,1023,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,1023,0.01924266666173935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,2047,0.028512001037597656
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,4095,0.01860800012946129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,4095,0.040607998768488564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,2047,0.011237333218256632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,8191,0.058917333682378135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,8191,0.024405332903067272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,16383,0.0865066647529602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,16383,0.041296000281969704
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,32767,0.12349866827329
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,32767,0.06270933151245117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,65535,0.19925334056218466
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,65535,0.1071519951025645
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,fp8,131071,0.193338672320048
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,8,4,128,1,float16,float16,131071,0.35657068093617755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,1,0.09037866195042928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,1,0.0918933351834615
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,3,0.0934986670811971
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,3,0.09463999668757121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,7,0.09757866462071736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,7,0.0985599954922994
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,15,0.12244266271591187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,15,0.12266133228937785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,31,0.1229759951432546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,63,0.12294399738311768
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,31,0.12241599957148235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,63,0.12333866953849792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,127,0.04665599763393402
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,127,0.01782400036851565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,255,0.05421866476535797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,255,0.024858665963013966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,511,0.03794133414824804
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,1,0.09053867061932881
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,511,0.0705866664648056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,1,0.09218666950861613
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,float16,1023,0.10570133725802104
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,1,128,1,float16,fp8,1023,0.05648000041643778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,3,0.09333333373069763
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,3,0.09505599737167358
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,7,0.09734400113423665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,7,0.09886399904886882
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,15,0.12269866466522217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,15,0.12283733487129211
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,31,0.12314132849375407
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,31,0.12288000186284383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,63,0.1295360028743744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,63,0.12334400415420532
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,127,0.08290666838486989
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,127,0.02959466725587845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,255,0.09496000409126282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,255,0.04168533285458883
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,1,0.09062400460243225
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,511,0.06142933170000712
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,511,0.12732799847920737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,fp8,1023,0.10168533523877461
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,2,128,1,float16,float16,1023,0.1949066718419393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,1,0.09265066186587016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,3,0.09397332866986592
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,3,0.09577600161234538
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,7,0.09757866462071736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,7,0.0995146632194519
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,15,0.12312533458073933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,15,0.12361600001653035
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,31,0.12841066718101501
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,31,0.12391466895739238
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,63,0.13202133774757385
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,63,0.12966932853062949
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,127,0.15343999862670898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,127,0.05601066847642263
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,255,0.17606399456659952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,255,0.07354666789372762
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,511,0.2379200061162313
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,511,0.11231999595959981
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,1,0.17483200629552206
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,1,0.1783519983291626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,float16,1023,0.3721439838409424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,3,0.1829920013745626
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,3,0.18055466810862222
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,8,4,128,1,float16,fp8,1023,0.18976000944773355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,7,0.18982400496800741
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,7,0.19230933984120688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,15,0.2390986680984497
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,15,0.23966399828592935
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,31,0.2395039995511373
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,31,0.24083733558654785
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,63,0.25083200136820477
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,63,0.24088533719380698
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,127,0.08562133709589641
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,127,0.033904001116752625
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,3,0.1808799902598063
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,1,0.17475199699401855
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,255,0.09821333487828572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,1,0.17898666858673096
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,3,0.18392000595728555
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,255,0.04706133405367533
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,float16,511,0.12957866986592612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,1,128,1,float16,fp8,511,0.06665066878000896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,7,0.190010666847229
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,7,0.19298666715621948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,15,0.24016000827153525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,15,0.2403200070063273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,31,0.250874658425649
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,63,0.25305066506067914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,31,0.24116800228754678
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,63,0.25328532854715985
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,127,0.15555733442306519
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,127,0.05723733206590017
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,255,0.17841066916783652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,1,0.17545600732167563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,255,0.07617599765459697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,3,0.18146665891011557
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,float16,511,0.24134933948516846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,1,0.17963733275731406
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,2,128,1,float16,fp8,511,0.11390399932861328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,3,0.18525866667429605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,7,0.1919040083885193
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,7,0.19460266828536987
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,15,0.24844799439112344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,15,0.24253867069880167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,31,0.25277332464853924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,31,0.2545973261197408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,63,0.25657065709431964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,63,0.25672000646591187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,127,0.29605867465337116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,1,0.009445333232482275
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,127,0.10475732882817586
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,1,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,3,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,3,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,255,0.1402133305867513
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,255,0.34191465377807617
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,7,0.009514666472872099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,15,0.009765333185593287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,7,0.009637333452701569
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,float16,511,0.46329601605733234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,15,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,8,4,128,1,float16,fp8,511,0.2155839999516805
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,31,0.010485333700974783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,31,0.0106133334338665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,63,0.012250666817029318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,63,0.01201066623131434
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,127,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,127,0.01239466667175293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,255,0.012554666648308435
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,255,0.012240000069141388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,1023,0.008810666700204214
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,511,0.007989333321650824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,511,0.012714666624863943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,1023,0.01332266628742218
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,2047,0.010656000425418219
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,2047,0.021909333765506744
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,8191,0.036517334481080375
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,4095,0.025333332518736523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,4095,0.012549333274364471
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,8191,0.017162666966517765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,16383,0.053530668218930565
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,16383,0.0288426677385966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,32767,0.08124266564846039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,32767,0.04160533348719279
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,65535,0.12753599882125854
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,65535,0.06956266860167186
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,1,0.009632000078757605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,1,0.010037333394090334
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,float16,131071,0.20993600289026895
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,3,0.009589333087205887
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,1,128,1,float16,fp8,131071,0.11622400085131328
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,7,0.009658666948477427
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,3,0.009904000287254652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,15,0.009829333052039146
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,15,0.010175999874869982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,31,0.010746666540702185
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,31,0.01080000028014183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,63,0.012335999558369318
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,63,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,127,0.0122079998254776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,127,0.012367999802033106
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,255,0.012634667257467905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,511,0.0129120002190272
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,1023,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,1023,0.019237333287795384
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,255,0.007285333548982938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,511,0.007738666608929634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,2047,0.028368001182874043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,2047,0.011114666859308878
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,4095,0.040789333482583366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,8191,0.024346667031447094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,8191,0.05852800110975901
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,4095,0.018592000007629395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,16383,0.08673066894213359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,16383,0.04215466479460398
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,32767,0.12285332878430684
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,32767,0.06292800108591716
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,65535,0.10745066404342651
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,65535,0.19902400175730386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,1,0.009839999799927076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,1,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,float16,131071,0.3555946747461955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,2,128,1,float16,fp8,131071,0.19449599583943686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,3,0.009599999835093817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,3,0.009808000177145004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,7,0.00960533320903778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,7,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,15,0.009877333417534828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,15,0.010159999753038088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,31,0.010666667173306147
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,31,0.010768000036478043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,63,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,127,0.010170666500926018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,63,0.012298667182525
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,127,0.006448000048597653
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,255,0.013546666751305262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,511,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,255,0.007402666533986728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,1023,0.014186666657527288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,511,0.008463999877373377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,1023,0.027930667002995808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,2047,0.043162668744723
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,2047,0.017445333302021027
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,4095,0.06771199901898702
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,4095,0.02638400097688039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,8191,0.03917866696914037
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,8191,0.08524800340334575
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,16383,0.12012799580891927
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,16383,0.0580213318268458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,32767,0.09902933239936829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,32767,0.19025067488352457
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,65535,0.3319466710090637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,65535,0.18016533056894937
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,1,0.009621333330869675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,1,0.00966933307548364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,3,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,float16,131071,0.6171040137608846
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,3,0.009914666414260864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,7,0.009695999945203463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,8,4,128,1,float16,fp8,131071,0.3427040179570516
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,7,0.00961599995692571
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,15,0.009850666547815004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,15,0.010015999898314476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,31,0.010501333822806677
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,63,0.012053333222866058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,31,0.010634666929642359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,63,0.01232533281048139
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,127,0.012479999413092932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,127,0.012373333175977072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,255,0.007167999943097432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,255,0.01267733300725619
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,511,0.008143999924262365
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,511,0.013466666142145792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,1023,0.01886933296918869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,1023,0.009088000282645226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,2047,0.02899733434120814
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,2047,0.01157333329319954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,8191,0.059477334221204124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,4095,0.04061333338419596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,4095,0.01850133389234543
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,8191,0.024666666984558105
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,16383,0.08827733000119527
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,16383,0.04196266829967499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,32767,0.12524799505869547
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,1,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,32767,0.06305600206057231
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,65535,0.20058133204778036
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,65535,0.10779733459154765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,1,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,3,0.00972800018886725
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,float16,131071,0.3600800037384033
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,3,0.009824000298976898
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,7,0.00996800015370051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,7,0.009557333464423815
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,15,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,1,128,1,float16,fp8,131071,0.1947093407313029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,15,0.009984000275532404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,31,0.010496000448862711
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,31,0.010778666784365972
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,63,0.012319999436537424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,63,0.012266666938861212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,127,0.010175999874869982
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,127,0.006224000205596288
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,255,0.01321600005030632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,255,0.007621333623925845
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,511,0.00855466661353906
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,1023,0.027653334041436512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,511,0.01945066700379054
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,1023,0.013989333063364029
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,2047,0.017237332959969837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,2047,0.04324266811211904
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,4095,0.06758933266003926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,4095,0.02661866694688797
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,8191,0.08534933129946391
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,8191,0.03920000046491623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,16383,0.058880001306533813
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,16383,0.12086932857831319
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,32767,0.19057067235310873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,32767,0.09884267052014668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,65535,0.18051199118296304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,65535,0.3317173322041829
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,1,0.00884799969693025
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,1,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,3,0.009018666421373686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,float16,131071,0.6170133352279663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,3,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,32,1,4,2,128,1,float16,fp8,131071,0.3428853352864583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,7,0.009082666908701261
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,7,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,15,0.00916800027092298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,15,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,31,0.009898666913310686
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,63,0.011616000284751257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,31,0.010058666889866194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,127,0.011792000383138657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,127,0.011648000528415045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,63,0.011621333658695221
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,255,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,255,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,511,0.016016000260909397
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,511,0.015882667154073715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,2047,0.01706133286158244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,1023,0.017370666066805523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,1023,0.017407999684413273
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,2047,0.017498667041460674
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,4095,0.017370666066805523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,4095,0.017525333911180496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,8191,0.017850667238235474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,8191,0.012175999581813812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,16383,0.019866666446129482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,16383,0.014746667196353277
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,32767,0.023813332120577495
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,32767,0.018624000251293182
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,65535,0.023557332654794056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,65535,0.028981332977612812
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,float16,131071,0.03939733405907949
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,1,0.008938666433095932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,1,128,1,float16,fp8,131071,0.031178665657838184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,3,0.00915733352303505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,3,0.009183999771873156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,1,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,7,0.00902399979531765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,7,0.009130666653315226
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,31,0.00997866690158844
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,15,0.009279999881982803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,15,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,31,0.010202666744589806
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,63,0.011711999773979187
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,63,0.011610666910807291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,127,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,127,0.01163200040658315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,255,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,255,0.011605333536863327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,511,0.016197333733240765
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,511,0.01605333387851715
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,1023,0.017152000218629837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,1023,0.017317333569129307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,2047,0.017397332936525345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,4095,0.015306666493415833
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,2047,0.01718933383623759
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,4095,0.010159999753038088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,8191,0.01869333287080129
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,8191,0.012437333663304647
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,16383,0.020202666521072388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,16383,0.014762666076421738
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,32767,0.018805333723624546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,32767,0.02463999887307485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,65535,0.02382933348417282
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,65535,0.030752000709374745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,float16,131071,0.0582826683918635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,3,0.008986666798591614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,1,0.008949333180983862
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1,1,4,2,128,1,float16,fp8,131071,0.036415999134381614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,7,0.009050666665037474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,3,0.009109333157539368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,7,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,15,0.00978133330742518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,15,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,63,0.011770666887362799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,31,0.01003200002014637
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,31,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,63,0.011541333049535751
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,127,0.011887999872366587
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,127,0.011605333536863327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,255,0.011658667276302973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,255,0.011760000139474869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,511,0.0161013330022494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,1023,0.01725333308180173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,511,0.016122666498025257
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,4095,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,1023,0.0173333336909612
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,2047,0.017418666432301205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,4095,0.010112000008424124
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,2047,0.017429333180189133
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,8191,0.012170666207869848
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,8191,0.019653332730134327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,16383,0.014479999740918478
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,16383,0.02103466788927714
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,32767,0.01877333347996076
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,32767,0.025311999022960663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,65535,0.03107733279466629
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,65535,0.02405333270629247
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,1,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,1,0.00919999989370505
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,float16,131071,0.05792533357938131
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,1,128,1,float16,fp8,131071,0.03552533437808355
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,7,0.008997333546479544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,3,0.008986666798591614
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,3,0.009189333145817121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,15,0.009343999748428663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,7,0.00933333362142245
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,31,0.010053333515922228
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,15,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,63,0.011792000383138657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,63,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,31,0.010048000141978264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,127,0.011855999628702799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,127,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,255,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,255,0.011642667154471079
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,511,0.016127999871969223
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,511,0.01632000009218852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,1023,0.017237332959969837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,1023,0.017263999829689663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,2047,0.015279999623696009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,2047,0.008874666566650072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,4095,0.01669866715868314
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,4095,0.010138666878143946
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,8191,0.019551999866962433
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,8191,0.012448000411192576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,16383,0.022218666970729828
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,16383,0.014912000546852747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,32767,0.03578133384386698
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,32767,0.02180800090233485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,65535,0.028160000840822857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,1,0.009072000160813332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,131071,0.07858133316040039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,float16,65535,0.04785599807898203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,3,0.008997333546479544
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2,1,4,2,128,1,float16,fp8,131071,0.042021334171295166
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,1,0.009119999905427298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,31,0.01166933278242747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,7,0.009098666409651438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,3,0.009237333511312803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,7,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,15,0.009872000043590864
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,31,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,15,0.009935999910036722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,63,0.011653333902359009
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,63,0.01166933278242747
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,127,0.01032533310353756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,255,0.007967999825874964
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,511,0.01933866615096728
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,255,0.013530666629473368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,127,0.006351999938488007
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,511,0.008639999975760778
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,1023,0.02794133375088374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,1023,0.014111999422311783
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,2047,0.01739199956258138
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,2047,0.043098668257395424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,4095,0.0671253353357315
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,4095,0.026736001173655193
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,8191,0.039850667119026184
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,8191,0.0855519970258077
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,16383,0.1222826639811198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,16383,0.05975999931494395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,1,0.009152000149091085
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,32767,0.1932906707127889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,3,0.00922133338948091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,32767,0.0992746651172638
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,1,0.009269333134094873
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,3,0.009488000224033991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,float16,65535,0.33693865935007733
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,1,128,1,float16,fp8,65535,0.18031466007232666
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,7,0.009493333597977957
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,31,0.011749333391586939
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,15,0.010079999764760336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,15,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,7,0.009232000137368837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,31,0.011866666376590729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,63,0.011978667229413986
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,63,0.011834666132926941
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,255,0.008325333396593729
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,127,0.010453333457310995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,127,0.006714666883150737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,255,0.02054399996995926
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,511,0.01332266628742218
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,511,0.02824000020821889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,1023,0.04538666705290476
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,1023,0.019871999820073444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,2047,0.06565866867701213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,8191,0.11981866757074992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,2047,0.025402667621771496
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,4095,0.08304533362388611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,4095,0.03833599885304769
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,8191,0.061039999127388
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,16383,0.19041067361831665
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,16383,0.09816533327102661
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,1,0.009056000038981438
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,32767,0.17922133207321167
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,1,0.009488000224033991
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,32767,0.3308853308359782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,3,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,3,0.009343999748428663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,fp8,65535,0.34188799063364667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,64,1,4,2,128,1,float16,float16,65535,0.6143306493759155
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,15,0.00949866697192192
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,7,0.009077333534757296
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,15,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,7,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,31,0.010255999863147736
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,31,0.010090666512648264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,63,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,255,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,63,0.011695999652147293
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,127,0.011722666521867117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,127,0.011744000017642975
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,255,0.011733333269755045
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,511,0.01632000009218852
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,511,0.01648533344268799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,1023,0.017450666675964992
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,1023,0.017290666699409485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,2047,0.016917333006858826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,2047,0.008922666932145754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,4095,0.01020800011853377
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,4095,0.017685333887736004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,8191,0.021338666478792827
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,8191,0.012442667037248611
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,16383,0.02351466566324234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,16383,0.014778666198253632
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,32767,0.03611200054486593
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,32767,0.021984001000722248
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,1,0.009194666519761086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,65535,0.02829333394765854
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,65535,0.05031466484069824
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,1,0.00927466650803884
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,float16,131071,0.07939733564853668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,1,128,1,float16,fp8,131071,0.041663999358812966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,3,0.009045333291093508
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,3,0.009429333110650381
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,7,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,7,0.009375999992092451
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,15,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,31,0.010117333382368088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,15,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,63,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,31,0.010288000106811523
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,63,0.01180800050497055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,127,0.011792000383138657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,255,0.01179733375708262
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,127,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,511,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,255,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,1023,0.013616000612576803
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,511,0.016688000410795212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,2047,0.017349333812793095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,1023,0.00790933333337307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,2047,0.009114666531483332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,4095,0.010575999816258749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,8191,0.02788266787926356
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,4095,0.01807466646035512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,8191,0.014266667266686758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,16383,0.03204799940188726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,16383,0.017258666455745697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,32767,0.05215999980767568
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,32767,0.024864000578721363
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,65535,0.07585066556930542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,1,0.01492799942692121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,65535,0.04637333254019419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,fp8,131071,0.0666186660528183
TRTLLM,0.20.0,NVIDIA H200,generation_attention,4,1,4,2,128,1,float16,float16,131071,0.1272426644961039
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,3,0.01515199989080429
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,3,0.015509333461523056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,7,0.015189333508412043
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,7,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,15,0.01646399994691213
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,15,0.016901332885026932
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,31,0.0199946661790212
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,31,0.019946667055288952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,63,0.019904000063737232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,127,0.007087999954819679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,63,0.020186666399240494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,255,0.020725333442290623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,255,0.008389333263039589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,511,0.02812800059715907
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,1023,0.020154666155576706
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,127,0.010618666807810465
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,511,0.013386666774749756
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,2047,0.0654720018307368
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,1023,0.04531733194986979
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,4095,0.08457066615422566
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,2047,0.025754667818546295
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,4095,0.03875733415285746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,8191,0.12077333529790242
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,8191,0.060959999759991966
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,16383,0.09853866696357727
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,16383,0.19326400756835938
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,3,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,1,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,1,0.014965333044528961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,3,0.01545599972208341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,float16,32767,0.3363200028737386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,7,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,15,0.01643199970324834
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,7,0.015461333096027374
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,15,0.016805333395799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,31,0.019978666057189304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,1,128,1,float16,fp8,32767,0.1806079943974813
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,31,0.019978666057189304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,127,0.01565333331624667
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,63,0.020015999674797058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,63,0.020128000527620316
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,255,0.030080000559488933
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,127,0.00701333334048589
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,255,0.013376000026861826
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,511,0.04618666569391886
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,1023,0.06140799820423126
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,511,0.019685332973798115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,1023,0.02515733242034912
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,2047,0.08375466863314311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,2047,0.03823466598987579
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,4095,0.11949333548545837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,4095,0.05795200169086456
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,8191,0.1920586625734965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,8191,0.09758933385213216
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,1,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,16383,0.33461864789326984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,16383,0.17876799901326498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,1,0.015066667149464289
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,3,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,3,0.015413332730531693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,float16,32767,0.6183626651763916
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,7,0.015696000307798386
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,15,0.019343999524911244
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,7,0.01598400001724561
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,15,0.019578666736682255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,128,1,4,2,128,1,float16,fp8,32767,0.34096535046895343
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,31,0.01929066702723503
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,31,0.019391999890406925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,63,0.019637333850065868
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,63,0.01950399950146675
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,255,0.013349333157142004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,127,0.015775999675194424
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,511,0.045994664231936135
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,127,0.007520000139872233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,255,0.029951999584833782
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,511,0.01995733380317688
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,1023,0.0612960010766983
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,1023,0.025600001215934753
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,2047,0.03866666555404663
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,2047,0.08435199658075969
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,4095,0.12115733822186787
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,4095,0.05780800183614095
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,1,0.014842666685581207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,1,0.015008000036080679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,float16,8191,0.19382399320602417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,3,0.015568000574906668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,1,128,1,float16,fp8,8191,0.09820800026257832
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,7,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,3,0.015210667004187902
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,7,0.016058667252461117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,15,0.01937066639463107
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,15,0.019402666638294857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,31,0.019424000134070713
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,63,0.019600000232458115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,127,0.02518933266401291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,31,0.019600000232458115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,63,0.019813333948453266
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,255,0.04889066517353058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,255,0.019600000232458115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,127,0.010960000256697336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,511,0.06440000236034393
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,511,0.02518933266401291
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,1023,0.03841600070397059
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,1023,0.08041066428025563
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,2047,0.12202133735020955
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,2047,0.058693334460258484
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,4095,0.19284266233444214
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,3,0.0271519993742307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,1,0.025973332424958546
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,4095,0.09708799918492635
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,1,0.026314665873845417
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,3,0.026954665780067444
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,float16,8191,0.33610133330027264
TRTLLM,0.20.0,NVIDIA H200,generation_attention,256,1,4,2,128,1,float16,fp8,8191,0.17839999993642172
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,7,0.027962667246659596
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,7,0.027935999135176342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,15,0.034789333740870156
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,15,0.034703999757766724
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,31,0.035205334424972534
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,31,0.034874667723973594
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,63,0.03517866631348928
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,63,0.035274667044480644
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,127,0.02536533276240031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,127,0.011098666737476984
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,255,0.04891733328501383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,255,0.01982933282852173
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,511,0.026202666262785595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,511,0.06469333171844482
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,1023,0.08118399977684021
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,1023,0.03875733415285746
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,2047,0.12221866846084595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,1,0.02608533451954524
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,2047,0.05862933397293091
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,1,0.02645866572856903
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,3,0.026848000784715016
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,3,0.027242665489514668
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,7,0.027957332630952198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,7,0.02824000020821889
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,float16,4095,0.19432532787322998
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,15,0.03490666548411051
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,1,128,1,float16,fp8,4095,0.09801066915194194
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,15,0.03499199946721395
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,31,0.03528533379236857
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,31,0.0349440003434817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,63,0.03545066714286804
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,63,0.035455999275048576
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,127,0.04474666714668274
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,127,0.015941333025693893
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,511,0.06889600058396657
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,255,0.02146666745344798
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,255,0.0525493323802948
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,511,0.033813332517941795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,1023,0.10281067093213399
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,1023,0.05348266661167145
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,1,0.009370666618148485
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,1,0.009514666472872099
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,2047,0.17724267641703287
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,2047,0.09347200393676758
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,3,0.00921066664159298
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,3,0.00973866693675518
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,7,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,7,0.009253333633144697
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,float16,4095,0.3126986622810364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,15,0.009690666571259499
TRTLLM,0.20.0,NVIDIA H200,generation_attention,512,1,4,2,128,1,float16,fp8,4095,0.17469867070515951
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,31,0.010079999764760336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,15,0.009653333574533463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,63,0.011909333368142446
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,31,0.010480000327030817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,63,0.012042666474978128
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,127,0.01210133358836174
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,127,0.011941333611806234
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,255,0.01184533288081487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,511,0.016517333686351776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,255,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,511,0.016314666718244553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,2047,0.016805333395799
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,1023,0.013466666142145792
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,1023,0.008042666440208754
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,2047,0.009216000015536943
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,4095,0.017866666118303936
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,4095,0.010693332801262537
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,8191,0.027136000494162243
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,8191,0.014389333625634512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,16383,0.032314665615558624
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,16383,0.01757866640885671
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,32767,0.025029333929220837
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,32767,0.05175999800364176
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,65535,0.04790399968624115
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,65535,0.07612266639868419
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,float16,131071,0.12652267018953958
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,1,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,3,0.009312000125646591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,1,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,1,128,1,float16,fp8,131071,0.0663679987192154
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,7,0.009381333366036415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,3,0.009813333551088968
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,7,0.009674666449427605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,15,0.009455999980370203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,15,0.009770666559537252
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,31,0.010197333370645842
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,31,0.010453333457310995
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,63,0.011930666863918304
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,63,0.012053333222866058
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,127,0.012117333710193634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,127,0.011946666985750198
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,255,0.01198400060335795
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,255,0.01221866657336553
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,511,0.007701333612203598
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,511,0.012693333129088083
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,1023,0.013450667262077332
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,1023,0.00820266641676426
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,2047,0.0220320001244545
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,4095,0.025370667378107708
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,4095,0.011786667009194693
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,2047,0.0102186668664217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,8191,0.036271999279658
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,8191,0.016143999993801117
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,16383,0.05364799996217092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,16383,0.027994667490323383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,32767,0.03948266555865606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,32767,0.08026133477687836
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,65535,0.12723732988039652
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,65535,0.06629866858323415
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,1,0.04780266682306925
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,1,0.048453330993652344
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,3,0.049584001302719116
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,3,0.050106664498647056
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,float16,131071,0.20916267236073813
TRTLLM,0.20.0,NVIDIA H200,generation_attention,8,1,4,2,128,1,float16,fp8,131071,0.11151466766993205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,7,0.05146133402983347
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,7,0.05186133086681366
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,31,0.06454400221506755
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,15,0.06436799963315327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,15,0.06437333424886067
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,31,0.06417066852251689
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,63,0.06504533191521962
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,63,0.06500266492366791
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,127,0.044778664906819664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,127,0.01594666639963786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,255,0.053173333406448364
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,255,0.02143999934196472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,511,0.06971733272075653
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,511,0.03379733363787333
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,1023,0.05403733253479004
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,1023,0.10388799508412679
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,1,0.04775466521581014
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,1,0.04864533245563507
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,3,0.04966933528582255
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,3,0.05053866902987162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,7,0.05180266499519348
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,7,0.05222400029500326
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,15,0.06438933312892914
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,15,0.06482666730880737
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,float16,2047,0.1759786605834961
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,31,0.06471466521422069
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,31,0.06505600114663442
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,1,128,1,float16,fp8,2047,0.09452266494433086
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,63,0.0651093324025472
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,127,0.08277866741021474
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,63,0.06762666503588359
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,127,0.028325334191322327
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,255,0.09477333227793376
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,255,0.04098666707674662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,511,0.1267466644446055
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,511,0.06084266801675161
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,1023,0.1941653291384379
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,1023,0.10060266653696696
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,1,0.09211200475692749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,1,0.09032533566157024
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,3,0.09346666932106018
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,3,0.09511466821034749
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,float16,2047,0.33389333883921307
TRTLLM,0.20.0,NVIDIA H200,generation_attention,1024,1,4,2,128,1,float16,fp8,2047,0.1813066601753235
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,15,0.12257599830627441
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,15,0.12273066242535909
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,7,0.09885866443316142
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,7,0.09760000308354695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,31,0.12290666500727336
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,31,0.12327466408411662
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,63,0.1302826702594757
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,63,0.12343466281890869
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,127,0.08294933537642162
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,127,0.02940266579389572
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,255,0.09499200185139973
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,255,0.04192533095677694
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,1,0.09086933732032776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,1,0.09258133172988892
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,511,0.12745066483815512
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,3,0.09399466713269551
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,3,0.09579199552536011
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,511,0.06154666841030121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,7,0.09777067104975383
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,7,0.09940800070762634
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,15,0.12345066666603088
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,15,0.12318399548530579
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,31,0.12919466694196066
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,31,0.1234879990418752
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,float16,1023,0.19562133153279623
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,1,128,1,float16,fp8,1023,0.10198400417963664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,63,0.1325546701749166
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,63,0.1300106644630432
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,127,0.15448533495267233
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,127,0.05606933434804281
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,255,0.17817066113154092
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,1,0.009354666496316591
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,255,0.07403199871381123
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,1,0.009685333197315535
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,3,0.009573333586255709
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,3,0.00943999985853831
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,7,0.009322666873534521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,15,0.009434666484594345
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,7,0.009674666449427605
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,511,0.2395146687825521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,511,0.11219732960065205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,15,0.009893333539366722
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,31,0.010319999729593595
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,31,0.010293333480755487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,63,0.012096000214417776
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,255,0.012373333175977072
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,127,0.012133333832025528
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,127,0.012154666086037954
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,255,0.012144000579913458
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,511,0.012634667257467905
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,1023,0.013151999562978745
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,511,0.007631999750932057
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,float16,1023,0.3724000056584676
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,1023,0.008330666770537695
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,2047,0.022005334496498108
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,2047,0.010293333480755487
TRTLLM,0.20.0,NVIDIA H200,generation_attention,2048,1,4,2,128,1,float16,fp8,1023,0.18984532356262207
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,4095,0.02521066615978877
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,4095,0.011973333855470022
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,8191,0.03613866617282232
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,8191,0.015930666277805965
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,16383,0.05353599786758423
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,16383,0.028079998989899952
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,32767,0.0812853326400121
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,32767,0.03948266555865606
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,65535,0.12788800398508707
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,65535,0.06726400057474773
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,1,0.009461333354314169
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,1,0.00985599992175897
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,float16,131071,0.21067732572555542
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,3,0.009653333574533463
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,3,0.009365333244204521
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,1,128,1,float16,fp8,131071,0.11186666289965312
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,7,0.009413333609700203
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,15,0.009648000200589498
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,7,0.00983466642598311
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,15,0.009994666402538618
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,31,0.010480000327030817
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,31,0.010533332824707031
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,63,0.012149333953857422
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,255,0.012533333152532578
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,63,0.01211200033624967
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,127,0.011936000237862269
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,127,0.012128000458081564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,255,0.0074506668994824094
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,511,0.013173333058754602
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,511,0.007791999727487564
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,1023,0.019088000059127808
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,1023,0.009103999783595404
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,2047,0.011034666250149408
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,2047,0.028815999627113342
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,4095,0.018618666877349217
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,4095,0.04054400076468786
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,8191,0.05931733548641205
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,16383,0.04094400008519491
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,16383,0.08701333403587341
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,8191,0.024080000817775726
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,32767,0.12408533692359924
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,32767,0.062463998794555664
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,65535,0.19898666938145956
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,65535,0.10706133643786113
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,fp8,131071,0.19383466243743896
TRTLLM,0.20.0,NVIDIA H200,generation_attention,16,1,4,2,128,1,float16,float16,131071,0.3569706678390503
