framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,8192,11.949641927083333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,7168,10.383888753255208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,10240,14.787690226236979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,12288,17.733453369140626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,5120,7.488903299967449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,4096,5.958341471354166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,6144,9.012600708007813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,16384,25.634084065755207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3584,5.289710998535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2048,3.002929178873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,3072,4.4577885945638025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1536,2.2326454162597655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,2560,3.6777557373046874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,256,0.5372074762980144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,128,0.44260800679524737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,1024,1.5217642466227213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,512,0.8390965143839517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,768,1.1581738789876304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,64,0.439243729909261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,65536,32,0.423305606842041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,8192,2.7751284281412762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,10240,3.7581108093261717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,12288,4.477208455403646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,16384,5.931305440266927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,7168,2.614269765218099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,6144,2.193206278483073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,5120,1.8007498423258466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3584,1.238274129231771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,4096,1.388302993774414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,3072,1.0941749572753907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2560,0.884388287862142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,768,0.2963242530822754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1024,0.3869056065877279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,1536,0.5447104136149089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,2048,0.7129322687784831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,512,0.21635947227478028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,256,0.12926506996154785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,128,0.11492373148600261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,64,0.10704747041066487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,32,0.10373439788818359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,10240,2.6472681681315104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,12288,3.0969088236490885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,16384,4.210264587402344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,8192,2.0224992116292317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,7168,1.791499710083008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,6144,1.5620554606119792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,4096,1.0249194463094076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,5120,1.261237335205078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3584,0.8928170522054037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2560,0.6467402776082356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,3072,0.7731818517049154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,2048,0.5213823954264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1024,0.27810239791870117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,1536,0.40099201202392576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,768,0.21745600700378417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,128,0.0868074655532837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,256,0.09834880034128825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,512,0.1598709265391032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,64,0.07968533039093018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,32,0.07864426771799723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,12288,2.7250996907552083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,16384,65536,26.004781087239582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,16384,3.5610102335611975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,10240,2.2541226704915363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,7168,1.5031370798746744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,8192,1.7737898508707683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,4096,0.8732618967692056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,6144,1.3193631490071616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,5120,1.0864351908365886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,12288,65536,17.86160888671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2560,0.549014409383138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3072,0.6586751937866211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,3584,0.7629418690999349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1024,0.23687893549601235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,1536,0.33818238576253257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,768,0.18814719518025716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,2048,0.4624917348225911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,128,0.0751039981842041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,256,0.0927466630935669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,512,0.1385632038116455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,64,0.06885013580322266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,32,0.07124799887339274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,10240,1.7392052968343097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,12288,2.1484054565429687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,16384,2.9091061909993488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,8192,1.4211551666259765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,7168,1.2329109191894532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,6144,1.0501941045125327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,4096,0.7171509424845378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,5120,0.8769866943359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3584,0.6189781188964844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2560,0.4499360084533691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,10240,65536,14.831299845377604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,3072,0.543946647644043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,2048,0.3629727999369303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,768,0.15032000541687013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1024,0.1951690673828125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,1536,0.27739413579305017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,512,0.10881919860839843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,256,0.06839146614074706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,128,0.05964906613032023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,64,0.05457280079523722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,32,0.054048001766204834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,10240,1.4758922576904296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,12288,1.8112682342529296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,16384,2.37639897664388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,8192,1.1998495737711588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,7168,1.0438058853149415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,6144,0.8843231836954752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,5120,0.7438570658365886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,4096,0.6012597401936849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3584,0.525490125020345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2560,0.3815039952596029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,3072,0.45333013534545896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,2048,0.30574827194213866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1024,0.16423254013061522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,1536,0.23577067057291665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,768,0.1295093297958374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,256,0.060745600859324136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,512,0.09636159737904867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,8192,65536,11.866822306315104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,128,0.053940268357594814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,64,0.0494762659072876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,32,0.04800639947255452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,10240,1.307366434733073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,12288,1.5650773366292319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,16384,2.1097461700439455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,8192,1.047987174987793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,7168,0.9223882675170898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,6144,0.7772544225056965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,5120,0.6517194747924805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,4096,0.5267690658569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3584,0.460757319132487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2560,0.33663358688354494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,3072,0.3991744041442871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,2048,0.2698197364807129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1024,0.14497493108113607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,1536,0.20834026336669922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,7168,65536,10.482452392578125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,768,0.11212800343831379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,512,0.08357760111490885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,256,0.05303146839141846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,128,0.05382826725641886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,64,0.0431818683942159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,32,0.04225279887517293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,10240,1.057804807027181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,12288,1.2711573282877604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,16384,1.7992159525553386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,8192,0.8513418833414713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,7168,0.7441535949707031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,6144,0.6556341171264648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,4096,0.43386240005493165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,5120,0.53886292775472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3584,0.3901258786519369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2560,0.28040428161621095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,3072,0.32984533309936526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,2048,0.23107199668884276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1024,0.12077546914418538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,1536,0.17336319287618002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,6144,65536,8.88962910970052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,768,0.09795093536376953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,256,0.04898666540781657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,512,0.07149013678232828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,128,0.04458666642506917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,32,0.03576639890670776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,64,0.03585066795349121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,10240,0.8563103993733725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,12288,1.0061397552490234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,16384,1.3276565551757813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,8192,0.6857066472371419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,7168,0.5897866566975911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,6144,0.5061504046122234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,4096,0.3434698740641276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,5120,0.43427413304646806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3584,0.3006623903910319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2560,0.22381439208984374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,3072,0.2614037354787191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,2048,0.17633706728617352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1024,0.09580907026926676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,1536,0.13879359563191732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,768,0.07590719858805338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,5120,65536,7.532645161946614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,256,0.037330134709676104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,128,0.03305813272794088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,512,0.05663679838180542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,64,0.033854933579762776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,32,0.030125866333643597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,10240,0.7506261189778646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,12288,0.8861471811930338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,16384,1.1777183532714843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,8192,0.6029696146647135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,7168,0.5217631975809733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,6144,0.4461824099222819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,4096,0.30369812647501626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,5120,0.3819690704345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3584,0.26599574089050293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,3072,0.23100266456604004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2560,0.19721813201904298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,2048,0.15647573471069337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,4096,65536,5.769136047363281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1024,0.0865237315495809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,1536,0.12072319984436035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,768,0.06734826564788818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,512,0.05221759875615438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,256,0.03351039886474609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,128,0.029414399464925127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,64,0.026778666178385417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,32,0.026958932479222614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,10240,0.6313269297281902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,12288,0.7562623977661133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,16384,1.0103829065958658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,8192,0.5073557217915853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,7168,0.4470325469970703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,6144,0.387444273630778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,5120,0.32203092575073244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,4096,0.26146133740743005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3584,0.23115199406941733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2560,0.1677567958831787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,3072,0.19823039372762044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,2048,0.13645226160685223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3584,65536,4.977151997884115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1024,0.07439146836598715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,768,0.05963306824366251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,1536,0.10487146377563476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,512,0.04620266755421956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,256,0.030434133609135945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,128,0.026177066564559936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,64,0.023628799120585124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,32,0.024980266888936363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,10240,0.5226506551106771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,12288,0.6287071863810222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,16384,0.8398080190022787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,8192,0.42003199259440105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,7168,0.3714336077372233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,6144,0.3212949434916178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,4096,0.21755946477254234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,5120,0.26704107920328773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3584,0.19225707054138183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2560,0.13908905982971193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,3072,0.16594667434692384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,2048,0.11482026576995849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,3072,65536,4.426640828450521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1536,0.08776960372924805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,1024,0.06314239899317423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,768,0.0506495992342631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,256,0.02746559977531433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,512,0.04044373432795207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,128,0.021978666385014854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,32,0.020124799013137816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,64,0.01961173415184021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,10240,0.4208415985107422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,12288,0.4972554524739583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,16384,0.662167485555013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,8192,0.3383904139200846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,7168,0.29381014506022135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,6144,0.255186128616333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,4096,0.1720245361328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,5120,0.2149376074473063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3584,0.15287787119547527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2560,0.11223573684692383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,3072,0.13146026929219562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,2048,0.0911242643992106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2560,65536,3.4244949340820314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1024,0.05036266644795736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,1536,0.07096746762593588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,768,0.04091946681340535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,512,0.032708267370859787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,128,0.016752000649770102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,256,0.022010666131973267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,64,0.014856533209482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,32,0.01564800043900808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,10240,0.3144949277242025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,12288,0.3761653264363607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,16384,0.4955349286397298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,8192,0.2532138665517171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,7168,0.22321386337280275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,6144,0.19103892644246417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,5120,0.16146987279256184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,4096,0.1309664011001587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3584,0.11484053134918212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2560,0.08491946856180826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,3072,0.10028159618377686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,2048,65536,2.7633344014485677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,2048,0.06928426424662272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1024,0.039221334457397464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,1536,0.0543498675028483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,768,0.033297065893809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,512,0.024709333976109825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,256,0.018100267648696898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,128,0.013808000087738036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,64,0.012414933244387309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,32,0.01245013376077016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,10240,0.20889600118001303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,12288,0.25683306058247884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,16384,0.3326805432637533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,8192,0.16803733507792157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,7168,0.15197547276814777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,6144,0.128220796585083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,5120,0.10737386544545491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,4096,0.08944746653238932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3584,0.07717653115590414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2560,0.05790079832077026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,3072,0.06653759876887003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,2048,0.046513064702351885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1536,65536,2.008238983154297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1024,0.025489066044489545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,1536,0.03703039884567261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,768,0.020721065998077392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,512,0.016170666615168253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,256,0.011195733149846395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,64,0.0076885332663853955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,128,0.008801066875457763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,32,0.00812906672557195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,10240,0.16509440739949544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,12288,0.2000864028930664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,16384,0.2594090620676676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,8192,0.13296106656392415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,7168,0.11906987031300861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,6144,0.10040000279744465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,4096,0.07001067002614339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,5120,0.08487146695454915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3584,0.06054293314615885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2560,0.0447711984316508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,3072,0.05369066794713338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,2048,0.03660693168640137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,1024,65536,1.3417802174886069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1536,0.02850666642189026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,1024,0.02009920080502828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,512,0.012684800227483115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,768,0.016376533110936484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,256,0.008980266253153483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,32,0.006628266473611195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,64,0.006381866832574208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,128,0.007259733478228251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,10240,0.11163199742635091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,12288,0.12997013727823895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,16384,0.1718623956044515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,8192,0.09014506340026855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,7168,0.07783679962158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,6144,0.06733653545379639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,4096,0.045925335089365645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,5120,0.057929599285125734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3584,0.040992001692454025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2560,0.031194667021433514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,3072,0.03570133447647095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,768,65536,1.0230400085449218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,2048,0.02481813430786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,768,0.011946666240692138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1024,0.01420906682809194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,1536,0.01933120091756185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,512,0.009562666217486065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,32,0.005629866818586985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,256,0.00724480003118515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,64,0.005486933390299479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,128,0.0061930666367212926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,10240,0.06620479822158813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,12288,0.07925972938537598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,16384,0.10269653002421061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,8192,0.05356800158818563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,7168,0.04785279830296834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,6144,0.04237653414408366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,4096,0.029649066925048827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,5120,0.035846400260925296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3584,0.026731733481089277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2560,0.020508799950281778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,3072,0.023896533250808715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,512,65536,0.6820789337158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,2048,0.013742933670679728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1024,0.008524800340334576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,1536,0.010838400324185688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,768,0.007429333527882893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,512,0.006201600035031637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,128,0.004496000210444133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,256,0.005130666494369507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,64,0.004082133372624716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,32,0.004307200014591217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,10240,0.05995306571324667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,12288,0.07087893486022949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,16384,0.10273706912994385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,8192,0.048761598269144696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,7168,0.04307093222935994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,6144,0.03873066504796346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,4096,0.026707200209299724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,5120,0.032332799832026166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3584,0.02396799921989441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2560,0.018272000551223754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,3072,0.021332265933354695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,256,65536,0.4046730677286784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,2048,0.009476266304651896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1024,0.006037333110968272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,768,0.0055189331372578945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,1536,0.007534933090209961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,512,0.004774400095144907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,64,0.0033621333539485933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,128,0.0035594666997591654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,256,0.004058666775623957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,32,0.0036373332142829893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,10240,0.05810453494389852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,12288,0.06903786659240722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,16384,0.09155840078989665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,8192,0.04726506471633911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,7168,0.04212906757990519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,6144,0.03710506757100423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,4096,0.025943466027577716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,5120,0.03168960014979045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3584,0.023196800549825033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,128,65536,0.4018122673034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2560,0.014155733585357665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,3072,0.020503467321395873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,2048,0.009946667154630025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1024,0.006603733201821645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,768,0.005378133555253347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,1536,0.007971199850241344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,512,0.004660266637802124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,128,0.003390933324893316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,256,0.003849600007136663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,64,0.003307733436425527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,32,0.00329066663980484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,10240,0.057766401767730714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,12288,0.06862506866455079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,16384,0.08915092945098876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,8192,0.04677120049794515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,7168,0.04159466822942098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,6144,0.03642773230870565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,5120,0.031241599718729657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,4096,0.02558186650276184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3584,0.02272319992383321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,64,65536,0.36526505152384436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2560,0.014413866400718688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,3072,0.020102399587631225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,768,0.00489279975493749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,2048,0.008685866991678875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1024,0.0053951998551686605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,1536,0.0068917334079742435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,512,0.00411520004272461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,256,0.0035573333501815797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,64,0.003053866575161616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,128,0.0031989333530267083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,32,0.0030026666820049284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,8192,5.863465372721354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8192,32,65536,0.35873600641886394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,10240,7.503822835286458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,12288,8.890478515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,6144,4.318121846516927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,5120,3.6567860921223962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,7168,5.010046895345052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,16384,12.09501953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,4096,2.806776428222656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1536,1.0922826131184897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2048,1.436132303873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,2560,1.8401087443033854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3584,2.533042144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,3072,2.2082667032877605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,1024,0.7381130854288738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,512,0.4264810562133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,768,0.5946207682291667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,256,0.27374080022176106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,128,0.2306901295979818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,32,0.2023914655049642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,65536,64,0.20788267453511558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,8192,1.4324811299641929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,10240,1.827245839436849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,12288,2.164483133951823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,16384,2.923200988769531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,7168,1.2412405649820963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,6144,1.0429930369059244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,5120,0.8703573226928711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,4096,0.7056021372477214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3584,0.617303466796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2560,0.4487658818562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,3072,0.5365546544392903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,768,0.14734400113423665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1024,0.19277226130167643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,1536,0.27665281295776367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,2048,0.3657493273417155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,512,0.10920533339182537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,256,0.06720853646596273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,128,0.05933546622594198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,64,0.05569173494974772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,32,0.05582506656646728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,10240,1.2982805887858073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,12288,1.5468692779541016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,16384,2.1206570943196614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,8192,1.0071690877278647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,7168,0.8931701024373373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,6144,0.7596330642700195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,4096,0.5165877342224121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,5120,0.6378389358520508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3584,0.4508693377176921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2560,0.32751998901367185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,3072,0.39191147486368816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,2048,0.26275946299235026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1024,0.14216426213582356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,1536,0.20223573048909507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,768,0.11173120339711506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,512,0.08388266563415528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,256,0.05241920153299967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,128,0.04667199850082397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,64,0.04267093340555827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,32,0.04369173447291057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,16384,65536,12.305223592122395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,12288,1.281338628133138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,16384,1.745460255940755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,10240,1.112447992960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,8192,0.85119997660319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,7168,0.7506858825683593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,5120,0.5384586970011394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,6144,0.6552864074707031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,4096,0.43663679758707685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,12288,65536,8.982711791992188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3584,0.383894411722819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,3072,0.3370368003845215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2560,0.28006292978922526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,2048,0.23156159718831382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1024,0.12403946717580158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,768,0.0973642667134603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,1536,0.1734421412150065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,512,0.07235413392384847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,64,0.0359605352083842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,128,0.03961813449859619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,256,0.04943466583887736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,32,0.03688746690750122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,10240,0.8391455968221029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,12288,1.0246602376302083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,16384,1.3624703725179037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,8192,0.6718805313110352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,7168,0.60382506052653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,6144,0.5121568044026692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,4096,0.35080000559488933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,5120,0.4256437301635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3584,0.3046282768249512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,10240,65536,7.4205078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2048,0.17904319763183593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,3072,0.26704533894856775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1536,0.13843092918395997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,2560,0.21926399866739907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,256,0.03721919854482015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,768,0.07809173266092936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,512,0.057051734129587806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,1024,0.09687039852142335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,128,0.032307199637095135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,64,0.033149866263071696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,32,0.030572799841562907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,10240,0.7341311772664387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,12288,0.8973834355672201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,16384,1.1627338409423829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,8192,0.5876757303873699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,7168,0.5287957191467285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,6144,0.4452831904093425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,4096,0.3076885223388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,5120,0.3725536028544108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3584,0.2651871999104818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,8192,65536,5.695861307779948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2560,0.19229973157246907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,3072,0.23423147201538086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,2048,0.15609067281087238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1536,0.12187519868214924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,768,0.06916800340016684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,1024,0.08435733318328857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,512,0.05047253370285034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,256,0.03336533308029175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,64,0.02688000003496806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,128,0.029089067379633588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,32,0.027133866151173906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,10240,0.6297130584716797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,12288,0.7713823954264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,16384,0.997047487894694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,8192,0.5058272043863933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,7168,0.45496212641398115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,6144,0.3820746739705404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,4096,0.26486825942993164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,5120,0.32072426478068033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3584,0.22814186414082846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,7168,65536,5.153383382161459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2560,0.1658090591430664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,3072,0.201638396581014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,2048,0.1343616008758545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1536,0.10635093053181965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,1024,0.07342720031738281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,768,0.05983999967575073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,512,0.04400746822357178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,256,0.029598933458328248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,128,0.025621332724889118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,64,0.024282666047414143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,32,0.024784000714619954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,10240,0.5288021405537923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,12288,0.6484021504720052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,16384,0.8405855814615885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,8192,0.4249919891357422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,7168,0.383460267384847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,6144,0.32261441548665365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,5120,0.2701504071553548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,4096,0.22395200729370118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3584,0.1929205258687337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,6144,65536,4.295406087239583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2048,0.11518399715423584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,2560,0.14431573549906412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,3072,0.1665983994801839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1536,0.09121493498484293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,1024,0.06337706645329794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,256,0.02799466649691264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,512,0.03885120153427124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,768,0.051744000116984046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,64,0.019938133160273232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,128,0.022567466894785563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,32,0.020086399714152017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,10240,0.4168042818705241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,12288,0.5013312021891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,16384,0.665340805053711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,8192,0.3352287928263346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,7168,0.29622294108072916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,6144,0.2558037281036377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,4096,0.17315413157145182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,5120,0.21322347323099772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,5120,65536,3.5973726908365884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3584,0.1535157362620036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,3072,0.13242133458455402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2560,0.11140800317128499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1536,0.07070720195770264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,2048,0.09321813583374024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,1024,0.0503114660580953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,768,0.04065066576004028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,512,0.0313045342763265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,256,0.02183039983113607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,128,0.01676266590754191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,32,0.01551040013631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,64,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,10240,0.39015251795450845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,12288,0.46941334406534835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,16384,0.6204213460286458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,8192,0.3136042594909668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,7168,0.2775146802266439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,6144,0.2387466589609782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,4096,0.16257279713948566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,5120,0.1996021270751953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,4096,65536,2.8211146036783856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3584,0.14322346051534016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2048,0.08904639879862467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,3072,0.12418239911397297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,2560,0.10451733271280925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1536,0.06674880186716715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,512,0.029165865977605183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,768,0.039493334293365476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,1024,0.04873600006103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,256,0.020064000288645426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,128,0.015170133113861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,64,0.014332800110181173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,32,0.015018666783968607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,10240,0.3238453229268392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,12288,0.37910188039143883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,16384,0.49945812225341796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,8192,0.26073493957519533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,7168,0.22486400604248047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,6144,0.19264747301737467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,4096,0.13227519989013672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,5120,0.1662314732869466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3584,65536,2.583487955729167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3584,0.1159050703048706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,3072,0.10120533307393391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2560,0.08714133103688558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,2048,0.07172586917877197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1536,0.05403199990590414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,256,0.017579734325408936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,1024,0.039345065752665206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,512,0.024921600023905435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,768,0.031983999411265056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,128,0.0136543999115626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,64,0.012181333700815837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,32,0.012498133381207784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,10240,0.26322666803995765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,12288,0.3137951850891113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,16384,0.41323413848876955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,8192,0.21162667274475097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,7168,0.18580586115519207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,6144,0.15880533854166667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,3072,65536,2.068003209431966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,4096,0.10886613527933757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,5120,0.13488000233968098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3584,0.09527786572774252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,3072,0.08348373572031656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2048,0.05805226564407349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,2560,0.0704213301340739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1536,0.04460800091425578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,1024,0.03172053297360738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,512,0.019793067375818887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,256,0.013610666990280152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,768,0.025813333193461102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,128,0.010551466544469198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,32,0.009408000111579894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,64,0.009248000383377076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,10240,0.2087242603302002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,12288,0.2473877271016439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,16384,0.32829440434773766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,8192,0.16817599932352703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,7168,0.14651093482971192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,6144,0.12680746714274088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,4096,0.08605120182037354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2560,65536,1.6892244974772137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,5120,0.10716479619344074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3584,0.07656746705373128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2048,0.04611733357111613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,2560,0.05800853172938029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,3072,0.06609280109405517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1536,0.035538132985432944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,1024,0.025481599569320678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,512,0.01602240006128947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,768,0.021065600713094077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,256,0.011289599537849426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,64,0.007542400062084198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,128,0.008849066495895386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,32,0.008124800026416778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,10240,0.16573333740234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,12288,0.19374186197916668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,16384,0.25519893964131674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,8192,0.13359999656677246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,7168,0.11510506470998128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,6144,0.09853333632151286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,5120,0.08517013390858968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,2048,65536,1.3387808481852213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,4096,0.06766719818115234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3584,0.05936106840769449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2048,0.035853866736094156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,2560,0.0443562666575114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,3072,0.0530346671740214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1536,0.027513599395751952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,512,0.012556800246238708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,1024,0.019973333676656088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,768,0.016445866227149962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,256,0.00897813340028127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,128,0.007281066477298736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,64,0.0064074665307998655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,32,0.00666133314371109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,10240,0.10583679676055908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,12288,0.12589333057403565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,16384,0.1698421319325765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,8192,0.08558399677276611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,7168,0.07508266766866048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,6144,0.06649066607157389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,4096,0.04459413290023804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1536,65536,1.016162109375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,5120,0.055010131994883214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3584,0.04035626649856568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2048,0.024119466543197632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,2560,0.028490666548411054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,3072,0.03420053323109944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1536,0.01924906571706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,1024,0.014217600226402283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,512,0.009447466333707173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,768,0.011780266960461933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,256,0.007196799914042156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,128,0.006106666723887126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,64,0.005475200215975444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,32,0.005605333546797434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,10240,0.08246186574300131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,12288,0.09857280254364013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,16384,0.130457599957784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,8192,0.06657386620839437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,7168,0.05880320072174072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,6144,0.05104426542917887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,1024,65536,0.6513919830322266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,4096,0.03510506550470988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,5120,0.0428928017616272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3584,0.03081279993057251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,3072,0.026298666000366212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2048,0.01879146695137024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,2560,0.022501333554585775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1536,0.015056000153223673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,1024,0.011125333110491435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,512,0.00772266685962677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,768,0.009544533491134644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,256,0.006138666470845541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,128,0.005151999990145365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,64,0.004660266637802124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,32,0.004915200173854828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,10240,0.057316267490386964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,12288,0.06863786379496256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,16384,0.0899242639541626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,8192,0.046288001537323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,7168,0.04118293523788452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,6144,0.03550399939219157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,768,65536,0.52259521484375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,4096,0.02491413354873657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,5120,0.03027733365694682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3584,0.02173653244972229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,3072,0.0192138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2048,0.01322773297627767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,2560,0.0157258669535319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1024,0.00848746697107951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,1536,0.010689066847165425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,512,0.006214400132497152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,768,0.007419733206431071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,256,0.005095466474692027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,128,0.004458666841189066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,64,0.00410453329483668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,32,0.004196266829967499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,10240,0.04682240088780721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,12288,0.05398186842600504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,16384,0.06194666624069214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,8192,0.03271893262863159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,7168,0.03184106747309367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,6144,0.028251733382542926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,512,65536,0.3536287943522135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,4096,0.019269333283106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,5120,0.025153066714604693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3584,0.017207467555999757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,3072,0.015731199582417806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2048,0.011012267072995503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,2560,0.013262933492660523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1536,0.00897920032342275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,1024,0.007149866720040639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,256,0.004114133367935816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,512,0.0050357331832249965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,128,0.003684266656637192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,768,0.00591893345117569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,64,0.0033834666013717652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,32,0.0035103999078273775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,10240,0.03402986526489258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,12288,0.03868800004323324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,16384,0.05872426827748617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,8192,0.02690773407618205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,7168,0.024069333076477052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,6144,0.021658666928609214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,256,65536,0.21445226669311523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,5120,0.01792853275934855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3072,0.009124267101287841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,4096,0.010468266407648722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,3584,0.009377066294352214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1024,0.004789333542188009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,1536,0.0056757330894470215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2560,0.008117333551247915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,2048,0.0068256000677744556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,768,0.004341333111127218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,512,0.003903999924659729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,256,0.0034783999125162757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,128,0.0031669333577156065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,64,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,32,0.003253333270549774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,10240,0.03518826564153035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,12288,0.040999468167622885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,16384,0.050929065545399985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,8192,0.02954559922218323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,7168,0.026791467269261675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,128,65536,0.19385706583658854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,6144,0.02099840044975281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,5120,0.015287466843922935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3072,0.009169066945711773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,4096,0.008654933174451191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,3584,0.007960533102353413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1024,0.0046528001626332605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,1536,0.00553173323472341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2048,0.007042133311430614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,2560,0.007677866518497467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,256,0.0033344000577926634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,128,0.0030954666435718536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,512,0.003769599894682566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,768,0.004204800228277842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,64,0.0030645333230495454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,32,0.002951466788848241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,10240,0.03377279837926229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,12288,0.03968106508255005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,16384,0.049917864799499514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,8192,0.028113067150115967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,7168,0.025410133600234985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,64,65536,0.18665493329366048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,6144,0.022773333390553794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3072,0.006857599814732869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,4096,0.008525866270065307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,3584,0.007939200103282928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,5120,0.014455466469128927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1536,0.00510506679614385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,1024,0.004161066561937332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2048,0.005767466624577841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,2560,0.006126933296521505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,768,0.003939199944337209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,256,0.0031925333042939507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,128,0.0029578665892283124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,512,0.0035103999078273775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,64,0.002872533351182938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,32,0.0029098667204380036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4096,32,65536,0.18079253832499187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,8192,2.837957255045573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,10240,3.443059285481771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,7168,2.4988502502441405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,12288,4.252254994710286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,6144,2.1438570658365887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,5120,1.725369644165039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3584,1.2083403269449868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,4096,1.395026143391927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,16384,5.889799499511719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1536,0.5424522399902344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,1024,0.3850037256876627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,3072,1.06582825978597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2048,0.7096000035603841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,2560,0.8804896036783854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,128,0.12797333399454752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,256,0.16255040168762208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,512,0.239028263092041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,768,0.293070920308431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,64,0.1161685307820638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,65536,32,0.11369813283284505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,8192,0.6732885360717773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,10240,0.8407189051310221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,12288,1.0136618932088217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,16384,1.3835221608479817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,7168,0.5915882746378581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,6144,0.5111861228942871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,5120,0.42743574778238935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,4096,0.34753173192342124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3584,0.30328960418701173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,3072,0.26315946578979493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2560,0.22122774124145508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,768,0.07655786673227946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1024,0.09670080343882242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,2048,0.1806335926055908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,1536,0.13668266932169598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,512,0.0568181316057841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,256,0.03757973512013753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,128,0.03262826601664225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,64,0.03028800090154012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,32,0.03133226633071899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,10240,0.6301247914632161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,12288,0.754043706258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,16384,1.0102666854858398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,7168,0.4437856038411458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,8192,0.5066186587015789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,6144,0.3865301450093587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,4096,0.2592672030131022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,5120,0.32048107782999674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3584,0.23035732905069986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,3072,0.19751893679300944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2560,0.1659434636433919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,2048,0.1361248016357422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1024,0.07346346378326415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,1536,0.1042250633239746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,768,0.05903146664301554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,128,0.025630933046340943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,512,0.044064001242319746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,256,0.029310933748881024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,64,0.02400746742884318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,32,0.024984532594680788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,16384,65536,6.1064397176106775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,12288,0.635149892171224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,16384,0.8531967798868815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,10240,0.5516245524088542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,8192,0.42729066212972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,7168,0.3787797292073568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,6144,0.3320874532063802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,12288,65536,4.318599446614583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,5120,0.2714901288350423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,4096,0.2210634708404541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3584,0.1986570676167806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2560,0.14182507197062175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,3072,0.1727882703145345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1536,0.09238399664560953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,2048,0.11708160241444905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,768,0.05164693196614584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,1024,0.0635968009630839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,512,0.03901013135910034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,256,0.027719465891520183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,128,0.022050132354100548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,64,0.01995413303375244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,32,0.020036266247431437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,10240,0.43207359313964844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,12288,0.502396806081136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,16384,0.670245361328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,8192,0.34705918629964194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,7168,0.29669758478800456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,6144,0.25743039449055993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,10240,65536,3.412230428059896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,4096,0.17375680605570476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,5120,0.2203317324320475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3584,0.15422933896382648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1536,0.0711903969446818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2048,0.09431893030802409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,3072,0.13588053385416668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,2560,0.11249279975891113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,1024,0.05057599941889444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,512,0.03190933267275493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,768,0.04131840070088704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,256,0.021800533930460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,128,0.016676266988118492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,64,0.014849066734313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,32,0.015559466679890952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,10240,0.3759648005167643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,12288,0.4396458625793457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,16384,0.586903444925944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,8192,0.3013514518737793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,7168,0.25942400296529133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,6144,0.22425386110941567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,8192,65536,2.7645782470703124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,4096,0.1520821412404378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,5120,0.19287467002868652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3584,0.13365866343180338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,3072,0.11779519716898601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2560,0.09603412946065268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,2048,0.07941226959228516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1536,0.059723734855651855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,1024,0.04192853371302287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,768,0.03322239915529887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,256,0.017326933145523072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,512,0.025016532341639204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,128,0.014643200238545737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,64,0.013596799969673157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,32,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,10240,0.31824000676472985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,12288,0.37923094431559246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,16384,0.505401611328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,8192,0.25620692571004233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,7168,0.22466880480448403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,7168,65536,2.4009300231933595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,6144,0.1948618729909261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,4096,0.13211733500162762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,5120,0.1635807991027832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3072,0.10187946955362956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,3584,0.1180842638015747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2048,0.07035306294759115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,2560,0.08585920333862304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1024,0.039389868577321366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,1536,0.05480320056279501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,768,0.032219733794530234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,512,0.02467093269030253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,256,0.017762132485707603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,128,0.01381226678689321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,64,0.012345600128173827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,32,0.012713600198427835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,10240,0.2651285330454508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,12288,0.31809492111206056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,16384,0.4233141263326009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,8192,0.2130634625752767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,7168,0.18825173377990723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,6144,0.16304532686869305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,6144,65536,2.0975264231363933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,4096,0.11029653549194336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,5120,0.13569173812866211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3072,0.08793493111928305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,3584,0.0976469357808431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2560,0.07150932947794596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1536,0.04657599925994873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,1024,0.03291520078976949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,2048,0.058278401692708336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,768,0.026253867149353027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,512,0.019926400979359944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,128,0.01071573297182719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,256,0.014089600245157877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,64,0.009307733178138733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,32,0.009423999985059103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,10240,0.21554133097330727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,12288,0.25216320355733235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,16384,0.3346890767415365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,7168,0.14917012850443523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,8192,0.17342400550842285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,5120,65536,1.6719146728515626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,6144,0.12887146472930908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3072,0.06736000378926596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,3584,0.07838826974232992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,5120,0.10823573271433513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,4096,0.08920959631601968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2048,0.04625386794408162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,2560,0.05712533394495646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1536,0.036183468500773114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,1024,0.02611946662267049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,768,0.021117866039276123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,512,0.01641279955705007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,256,0.011718400319417318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,128,0.008776533603668212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,64,0.007730133334795634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,32,0.008057599763075511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,10240,0.2018890698750814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,12288,0.23834452629089356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,16384,0.3253856023152669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,4096,65536,1.317479451497396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,7168,0.14122880299886068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,8192,0.16249173482259113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,6144,0.12550506591796876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,5120,0.10388053258260091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3584,0.07420907020568848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,4096,0.08288319905598959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,3072,0.06521386702855428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1536,0.03510080178578694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,1024,0.02508373260498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2048,0.04446933269500732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,2560,0.05383786757787069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,256,0.011145599683125814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,128,0.008474666873613994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,768,0.02102933327356974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,512,0.0160181333621343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,64,0.007338666419188182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,32,0.007863466441631318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,10240,0.16387947400410968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,12288,0.19638293584187824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,16384,0.2647893269856771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3584,65536,1.296600596110026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,8192,0.13110400040944417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,6144,0.10211946964263915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,7168,0.11637439727783203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,5120,0.08500906626383463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,4096,0.06780800024668375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3584,0.061419733365376795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,3072,0.05233493248621622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2560,0.04400639931360881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,2048,0.03584853410720825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1536,0.028391466538111372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,1024,0.020080000162124634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,768,0.016681599617004394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,256,0.008869333068529765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,512,0.012998400131861368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,128,0.007311999797821045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,64,0.006412800153096516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,32,0.006588799754778545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,10240,0.13438186645507813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,12288,0.15726613998413086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,16384,0.20857067108154298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,3072,65536,1.023150952657064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,6144,0.08101973533630372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,7168,0.09357333183288574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,8192,0.10855147043863933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,5120,0.06829760074615479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3584,0.048903465270996094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,4096,0.05551253159840902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,3072,0.04347946643829346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2560,0.036008532842000326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1536,0.023576533794403075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,2048,0.02965013384819031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,1024,0.017401599884033205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,768,0.014740266402562461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,256,0.00834986666838328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,128,0.006853333115577698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,512,0.011111467083295187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,64,0.0061749334136645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,32,0.006448000172773997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,10240,0.10495146910349529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,12288,0.12451093196868897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,16384,0.16557547251383464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2560,65536,0.8177354812622071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,8192,0.08474240303039551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,7168,0.07425066630045572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,6144,0.06471466620763143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,5120,0.054459734757741296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3072,0.034227200349171955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,3584,0.038856534163157146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,4096,0.04425386587778728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2560,0.028844799598058062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1024,0.01448853313922882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,1536,0.019428267081578573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,2048,0.02394346594810486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,512,0.0095360000928243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,256,0.007180800040562947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,128,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,768,0.011709866921106975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,64,0.005508266886075338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,32,0.005643733342488607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,10240,0.08139839967091879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,12288,0.09748160044352214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,16384,0.1280735969543457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,2048,65536,0.6663306554158528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,8192,0.06550933519999186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,7168,0.0582368016242981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,6144,0.050190933545430506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,5120,0.04354879856109619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,4096,0.03426346778869629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3584,0.0303872009118398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,3072,0.026411734024683636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2560,0.02286720077196757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,2048,0.019025067488352455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1536,0.015245866775512696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,1024,0.01135040024916331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,512,0.007747200131416321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,768,0.009469866752624512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,256,0.006055466830730438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,128,0.005155199766159057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,64,0.004631466666857402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,32,0.004827733337879181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,10240,0.057307732105255124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,12288,0.0663477341334025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,16384,0.08743786811828613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1536,65536,0.5089578628540039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,8192,0.0460863987604777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,6144,0.034628268082936606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,7168,0.039858134587605794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,5120,0.02948906620343526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,4096,0.02427519957224528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3072,0.01840426723162333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,3584,0.020678400993347168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2560,0.015758933623631795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1024,0.008322133123874665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,2048,0.013408000270525614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,1536,0.010853333274523418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,768,0.007245866457621257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,128,0.0043935999274253845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,256,0.005070933202902476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,512,0.00613013356924057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,64,0.003987200061480204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,32,0.004222933451334635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,12288,0.052818131446838376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,10240,0.04443946679433187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,16384,0.06863040129343669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,1024,65536,0.3405824025472005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,7168,0.032307199637095135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,8192,0.035844266414642334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,6144,0.030482133229573567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,5120,0.02863146662712097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,4096,0.018590933084487914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3584,0.01853653391202291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2560,0.014295466740926108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,3072,0.016032000382741295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1536,0.009592533111572266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,1024,0.007249066730340321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,768,0.006246399879455566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,2048,0.012014933427174886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,256,0.004593066871166229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,512,0.005358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,64,0.0037109332780043284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,128,0.0039989332358042395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,32,0.003882666677236557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,12288,0.048647467295328775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,10240,0.04473066727320353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,16384,0.05244906743367513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,768,65536,0.2710890769958496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,8192,0.030910933017730714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,7168,0.029959466060002642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,6144,0.026070400079091387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,5120,0.023063466946283976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3584,0.015714133779207863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,4096,0.0172437330087026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,3072,0.014389333128929139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2560,0.01183786690235138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1024,0.006167466441790262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,768,0.005555200080076853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,2048,0.010115200281143188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,1536,0.008203733464082081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,128,0.0036544000109036768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,512,0.00472320020198822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,256,0.0040522667268912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,64,0.0033930666744709016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,32,0.003479466587305069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,10240,0.03009706735610962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,12288,0.030818132559458415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,512,65536,0.18351359367370607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,16384,0.03794453144073486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,7168,0.01821440060933431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,8192,0.02164906660715739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,5120,0.015432533621788026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,6144,0.0175872008005778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3072,0.00897706647713979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,3584,0.010346666971842448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,4096,0.010179199775060018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2560,0.007712000111738841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,768,0.004353066782156626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1536,0.006073600053787232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,2048,0.006896000107129414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,1024,0.004824533561865489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,512,0.003947733342647553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,256,0.0035232000052928926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,128,0.003201066702604294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,64,0.003066666672627131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,32,0.0032127998769283296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,12288,0.025573333104451496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,10240,0.022205867369969687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,256,65536,0.1144320011138916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,16384,0.03188160061836243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,7168,0.012481066584587096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,8192,0.013756799697875976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,6144,0.015033599734306336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,5120,0.014251733819643656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,4096,0.010796800255775452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2560,0.00699839989344279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3072,0.009063466389973959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,3584,0.009821866949399311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,2048,0.006285866598288219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,768,0.004214400053024292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1024,0.0046186665693918865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,1536,0.005460266768932342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,256,0.00335359995563825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,64,0.002906666696071625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,512,0.003755733370780945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,128,0.0030805334448814393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,32,0.003019733230272929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,128,65536,0.10448746681213379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,12288,0.023819732666015624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,10240,0.01994133392969767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,16384,0.029201066493988036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,8192,0.012382933497428894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,5120,0.009063466389973959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,7168,0.01016960044701894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,6144,0.015224533279736838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2560,0.006409599880377452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3584,0.007428266604741414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,4096,0.00832426647345225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,3072,0.0069013332327206925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1024,0.004227200150489807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,2048,0.005598933498064677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,1536,0.004946133494377137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,768,0.0038805333276589714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,512,0.003551999976237615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,256,0.0032458665470282235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,128,0.0029077333708604175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,64,0.0028618666032950084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,32,0.0029045333464940387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,10240,0.017128533124923705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,64,65536,0.0986687978108724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,12288,0.02291626731554667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,16384,0.02815893292427063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,5120,0.00826453318198522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,7168,0.009981866677602131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,6144,0.008835200468699138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,8192,0.010725333293279012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2560,0.006239999830722809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,4096,0.007517866790294647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3072,0.0068341334660847975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,3584,0.007177599767843883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1536,0.004862933357556661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,2048,0.005554133156935374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,768,0.003917866696914038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,1024,0.0041792000333468115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,256,0.00322026660044988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,512,0.003571200122435888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,64,0.0028927999238173166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,128,0.003067733347415924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,32,0.0029205332199732465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2048,32,65536,0.09561493396759033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,8192,1.4430795033772787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,7168,1.2251285552978515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,10240,1.8064117431640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,12288,2.119176483154297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,5120,0.8910346349080405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,4096,0.726416015625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,6144,1.0629440307617188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3584,0.6294272104899089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,16384,2.921771748860677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2048,0.374177074432373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,2560,0.46695680618286134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,3072,0.5431786855061849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1536,0.28366400400797526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,1024,0.2043626626332601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,256,0.08765652974446615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,128,0.06366613308588663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,64,0.06648853222529093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,512,0.1163530667622884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,768,0.15488746960957844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,65536,32,0.05755199988683065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,8192,0.3427199999491374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,10240,0.4259488105773926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,12288,0.5214005470275879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,16384,0.6811744054158528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,7168,0.30118080774943035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,6144,0.25969386100769043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,5120,0.22274239857991537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,4096,0.177459192276001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3072,0.1348405361175537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,3584,0.1556320031483968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2560,0.11650453408559162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1024,0.05161813497543335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,768,0.04208960135777791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,2048,0.09418666362762451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,1536,0.07239360014597575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,256,0.022486400604248048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,512,0.03182186683019002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,128,0.017573332786560057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,64,0.015195733308792115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,32,0.015453867117563882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,10240,0.32659947077433266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,12288,0.38200534184773766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,16384,0.5086965243021647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,7168,0.22666559219360352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,8192,0.26268906593322755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,6144,0.1959701379140218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,4096,0.13562026023864746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,5120,0.16393386522928874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3584,0.11800213654836018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2560,0.0865834633509318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,3072,0.10416533152262371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,2048,0.07155733108520508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1024,0.04049386580785115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,768,0.03240533272425334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,1536,0.05532480080922445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,256,0.017797333002090455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,512,0.025270400444666545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,128,0.014111999670664468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,64,0.012345600128173827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,32,0.012642133235931396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,16384,65536,2.8126612345377606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,12288,0.3293984095255534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,10240,0.27312854131062825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,16384,0.42925332387288406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,8192,0.22095999717712403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,6144,0.16561706860860187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,7168,0.18978667259216309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,12288,65536,2.0478026072184248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3584,0.09847573439280191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,5120,0.1412160078684489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,4096,0.11247573693593342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,3072,0.08887786865234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1536,0.047072001298268634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,1024,0.034959999720255534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2560,0.07216746807098388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,2048,0.06048426628112793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,768,0.026943999528884887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,512,0.020317866404851278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,256,0.014215466380119324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,128,0.011018666625022887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,32,0.009504000345865887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,64,0.009371733665466309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,10240,0.21376320521036782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,12288,0.2537504037221273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,16384,0.3439541180928548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,8192,0.1722965399424235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,7168,0.15047680536905925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,6144,0.13307092984517416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,10240,65536,1.710423405965169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,4096,0.08835519949595133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,5120,0.11001386642456054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3584,0.0799775997797648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,3072,0.06982293128967285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2048,0.047704533735911055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1536,0.03783040046691895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,2560,0.057366398970286045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,1024,0.02693439920743306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,128,0.00895360012849172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,512,0.016251732905705772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,768,0.020999467372894286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,256,0.01199893355369568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,64,0.007764266431331634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,32,0.007990399996439617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,10240,0.2073919932047526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,12288,0.24246932665506998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,16384,0.319975471496582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,8192,65536,1.337075169881185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,7168,0.1437930742899577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,8192,0.16694293022155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,6144,0.12375360329945881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3072,0.06429440180460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,4096,0.08425172964731852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,3584,0.07613440354665121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,5120,0.10538453261057537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2560,0.05458026727040609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,2048,0.04512106577555339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1024,0.025271467367808026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,1536,0.0353983998298645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,256,0.011541333794593812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,512,0.016040533781051636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,768,0.020719999074935914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,128,0.00842026670773824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,64,0.007277866701285045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,32,0.007938133180141449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,10240,0.16538346608479818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,12288,0.19630506833394368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,16384,0.26761385599772136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,8192,0.13279893398284912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,7168,65536,1.314017105102539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,7168,0.11637866497039795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,6144,0.1034816026687622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,4096,0.07032426993052164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,5120,0.0849781354268392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3584,0.06045973300933838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,3072,0.05391573508580526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2048,0.03665493329366048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,2560,0.044999468326568606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1024,0.02057173252105713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,1536,0.028189865748087566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,768,0.016581333676973977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,512,0.01318933367729187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,256,0.00916373332341512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,128,0.007235200206438701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,64,0.006435200075308482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,32,0.006681600213050842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,10240,0.13414613405863446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,12288,0.15926292737325032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,16384,0.21011734008789062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,6144,65536,1.0414453506469727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,8192,0.10813120206197102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,7168,0.09497066338857016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,6144,0.08176853656768798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,5120,0.07023466428120931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3072,0.04301226536432902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,4096,0.05642133156458536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,3584,0.049713067213694256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2048,0.030199466149012248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1024,0.017758933703104655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,2560,0.03661866585413615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,1536,0.023860265811284385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,768,0.01461013356844584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,512,0.011813333630561829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,256,0.008378666639328004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,128,0.007045333087444305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,64,0.006208000083764395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,32,0.006562133133411407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,10240,0.10623040199279785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,12288,0.12608319918314617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,16384,0.16840319633483886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,5120,65536,0.8503861109415689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,8192,0.08575146993001302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,6144,0.065611732006073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,7168,0.07518400351206461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,5120,0.05710613330205282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3072,0.03439573446909587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,3584,0.039869864781697587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,4096,0.0447925329208374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2560,0.029876265923182172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1536,0.019250132640202842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,2048,0.024106667439142863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,1024,0.014636799693107605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,768,0.012227200468381246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,512,0.009956266482671101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,256,0.007214933137098948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,128,0.006041599810123444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,64,0.005445333321889242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,32,0.0056650668382644655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,10240,0.1006229321161906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,12288,0.12178880373636883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,16384,0.15782400767008464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,4096,65536,0.6577536265055339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,5120,0.052502401669820155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,8192,0.08140373229980469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,7168,0.07302506764729819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,6144,0.05936533212661743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,4096,0.039843201637268066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2560,0.025921066602071125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3584,0.035743999481201175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,3072,0.03126399914423625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,2048,0.021249065796534218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1536,0.01699519952138265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,1024,0.012410666545232136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,768,0.010372266173362732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,256,0.006077866752942403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,512,0.007863466441631318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,128,0.005300266544024149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,64,0.004962133367856344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,32,0.005218133330345154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,12288,0.10107200145721436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,10240,0.08283519744873047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,16384,0.13078827063242596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3584,65536,0.6157354354858399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,5120,0.04335466623306274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,7168,0.05882560014724732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,6144,0.05106026728947958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,8192,0.06852479775746664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,4096,0.03476586739222209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3584,0.031089067459106445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,3072,0.027138133843739826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2560,0.023515733083089192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,2048,0.018987733125686645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1024,0.011777066191037496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,1536,0.015379200379053751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,768,0.009941333532333374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,512,0.008077866832415263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,256,0.0061141331990559895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,128,0.005245866874853769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,64,0.004668800036112467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,32,0.004881066580613455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,10240,0.0707530657450358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,12288,0.08629120190938314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,3072,65536,0.5117631912231445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,16384,0.1113749345143636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,5120,0.03745813369750976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,7168,0.051957333087921144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,6144,0.044674134254455565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,8192,0.05717013279596964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,4096,0.03054400086402893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2560,0.020356265703837077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3584,0.027702399094899494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,3072,0.023639466365178427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,2048,0.017391999562581383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1024,0.011096533139546711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,768,0.009327999750773112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,1536,0.0142794668674469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,512,0.0077237332860628765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,256,0.005899733304977417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,64,0.004556799928347269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,128,0.00506986677646637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,32,0.004826666911443075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,12288,0.06610666513442993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,10240,0.05554986794789633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2560,65536,0.43681494394938153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,16384,0.08680640061696371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,8192,0.04472746849060059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,7168,0.03975253502527873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,6144,0.03528853257497151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,5120,0.02908373276392619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3584,0.020935465892155967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,4096,0.023651200532913207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,3072,0.01845759948094686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2560,0.015877333283424378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1024,0.00849173367023468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,1536,0.011080533266067505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,2048,0.013497599959373474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,768,0.007216000060240428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,512,0.006169599791367849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,128,0.00445119986931483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,256,0.005083733300367991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,64,0.003982933362325033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,32,0.004167466859022776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,2048,65536,0.34989760716756185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,10240,0.043518932660420735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,12288,0.05114133358001709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,16384,0.06721386909484864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,8192,0.03591573238372803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,7168,0.03099306623140971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,6144,0.02674773335456848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,5120,0.022361600399017335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,4096,0.018573866287867228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3072,0.014531200130780539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,3584,0.01649173299471537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2560,0.012732799847920737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,2048,0.011187199751536052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1536,0.008796800176302593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,1024,0.007044266661008198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,768,0.006266666452089946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,512,0.005339733262856802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,256,0.0045056000351905824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,128,0.004006399959325791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,64,0.0036447999378045404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,32,0.0038015998899936674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1536,65536,0.26631253560384116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,12288,0.04313173294067383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,10240,0.03805439869562785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,16384,0.05190826654434204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,8192,0.03211413423220317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,7168,0.026397866010665894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,5120,0.022668800751368203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,6144,0.024420267343521117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,4096,0.017280000448226928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3584,0.014644267161687216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,3072,0.013848533233006796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2560,0.012619733810424805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,2048,0.00986346701780955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1536,0.0077802668015162155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,1024,0.006098133325576782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,768,0.005485866467158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,512,0.00472320020198822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,256,0.004040533304214477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,128,0.0036458666125933326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,32,0.0034389334420363107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,64,0.0033887999753157297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,1024,65536,0.1773802598317464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,12288,0.040948267777760824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,10240,0.032529066006342575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,16384,0.041289599736531575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,8192,0.02637866735458374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,5120,0.017986132701237997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,7168,0.02328426639238993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,4096,0.014308266838391624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,6144,0.021307732661565146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3584,0.013022933403650919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,3072,0.011859200398127238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2560,0.009246933460235595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,2048,0.008358400066693623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1024,0.00553706685702006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,1536,0.0070154666900634766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,768,0.004770133395989736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,512,0.004218666752179464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,256,0.0036831999818483984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,128,0.0033557333052158357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,64,0.0031360000371932983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,65536,0.1388330618540446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,768,32,0.003239466746648153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,12288,0.029577600955963134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,8192,0.021260799964269002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,10240,0.02581973274548848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,16384,0.03302719990412394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,6144,0.015844266613324484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,4096,0.011016533772150675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,7168,0.01725226640701294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,5120,0.013825066884358726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3072,0.0086517333984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,3584,0.009991466999053955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2048,0.006488533318042755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,2560,0.007681066791216533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1024,0.004871466755867004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,1536,0.005693866809209188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,768,0.004347733159859975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,512,0.0038965334494908653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,256,0.0034858666360378264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,128,0.003222399950027466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,64,0.003127466638882955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,32,0.0031413334111372627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,512,65536,0.09873387018839518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,8192,0.015044266978899637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,16384,0.02293973366419474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,12288,0.020360533396402994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,10240,0.017053866386413576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,7168,0.0137472003698349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,6144,0.013448533415794373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,4096,0.00888320008913676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,5120,0.012077866991360982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3584,0.008829866846402485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,3072,0.007795199751853943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2560,0.007086933155854543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,2048,0.006468266745408376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1536,0.005585066477457682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,1024,0.004724266628424326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,512,0.0037802666425704955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,768,0.004243200023969015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,256,0.003340800106525421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,128,0.0030559999247392017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,65536,0.06652479966481527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,64,0.0029397333661715193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,256,32,0.0029130667448043824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,8192,0.008636800448099773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,12288,0.011569066842397054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,10240,0.010318932930628459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,16384,0.014419200023015341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,6144,0.00862613320350647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,4096,0.008169599870840708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,7168,0.012168533603350322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,5120,0.009035733342170716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3072,0.007975466549396515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,3584,0.00862506628036499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2048,0.005636266867319743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,2560,0.006337066491444905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1024,0.00420053352912267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,768,0.003960533440113068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,1536,0.004932266473770141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,512,0.003562666724125544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,65536,0.05693013270696005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,128,0.0029877332349618276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,64,0.002796799937884013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,256,0.0031648000081380212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,128,32,0.0029663999875386557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,10240,0.008992000420888265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,8192,0.0077248002092043565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,12288,0.009961600104967754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,16384,0.012072533369064331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,7168,0.007539199789365132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,4096,0.007396266857783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,6144,0.008825600147247314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,5120,0.008135466774304708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3584,0.007141333321730297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,3072,0.006836266815662384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2560,0.006248533229033152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,2048,0.0054848000407218935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1536,0.004806399842103322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,1024,0.004154666761557261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,768,0.0038399999340375268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,512,0.0034986667335033415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,256,0.0030826665461063385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,65536,0.052758399645487467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,128,0.0029472000896930695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,64,0.002985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,64,32,0.002915200094381968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,8192,0.007577600081761678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,10240,0.008419199784596761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,12288,0.00925546685854594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,16384,0.010694400469462077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,4096,0.006612266600131989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,7168,0.007277866701285045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,5120,0.0069482664267222095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,6144,0.006635733445485433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3584,0.0069482664267222095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2560,0.006062933305899302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,3072,0.006594133377075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,2048,0.005432533224423727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1536,0.004750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,768,0.003717333326737086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,512,0.003409066547950109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,1024,0.004057600100835165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,65536,0.05015893379847208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,256,0.0031360000371932983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,64,0.0027647999425729113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,128,0.002833066632350286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1024,32,32,0.0027872001131375628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,7168,0.9199616114298502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,8192,1.0600917180379232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,10240,1.3143391927083334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,12288,1.5782240549723308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,5120,0.6609343846638998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,6144,0.7864959716796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,4096,0.5269290606180828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,16384,2.1246879577636717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3584,0.46898454030354814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2048,0.27367146809895837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,2560,0.3383925437927246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,3072,0.4047690709431966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1024,0.1477663993835449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,1536,0.21937920252482096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,512,0.08712106545766195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,128,0.04985920190811157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,256,0.06411306858062744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,768,0.11583253542582195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,64,0.050962134202321374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,65536,32,0.045413335164388016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,8192,0.25794879595438636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,10240,0.32697601318359376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,12288,0.38055572509765623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,16384,0.510371208190918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,6144,0.19977067311604818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,7168,0.2266826629638672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,5120,0.16330240567525228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,4096,0.1340640068054199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2560,0.08620266914367676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3072,0.1020469347635905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,3584,0.1201365311940511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,768,0.03319573402404785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1024,0.039843201637268066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,2048,0.0719210704167684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,1536,0.05544106562932333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,512,0.02574719985326131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,256,0.018045866489410402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,128,0.014042666554450989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,64,0.012230400244394939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,32,0.013008000453313193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,10240,0.24211947123209634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,12288,0.28160212834676107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,16384,0.37759574254353845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,8192,0.19494825998942059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,7168,0.1667957305908203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,6144,0.14554239908854166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,4096,0.09810026486714682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,5120,0.1242293357849121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3584,0.08741013209025064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2560,0.06551253398259481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,3072,0.07485546271006266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,2048,0.05288106600443522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1024,0.02911146680514018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,768,0.023563732703526817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,1536,0.0419050653775533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,512,0.01801919937133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,256,0.012958932916323343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,128,0.010007466872533162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,64,0.00853013296922048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,32,0.008778666456540424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,16384,65536,2.14966303507487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,12288,0.2466495990753174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,16384,0.3234442710876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,10240,0.20678399403889974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,8192,0.165829340616862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,6144,0.12492586771647136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,7168,0.14401920636494953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,12288,65536,1.5065184275309245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,4096,0.08518719673156738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,5120,0.10586559772491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3584,0.07501546541849771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,3072,0.0666592001914978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1024,0.02650880018870036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,1536,0.03556266625722249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2048,0.04612160126368205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,2560,0.055569068590799964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,768,0.02143146594365438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,512,0.016288000345230102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,256,0.011640533804893494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,128,0.010699733098347982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,64,0.009976533055305482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,32,0.01029973328113556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,10240,0.16332267125447592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,12288,0.19564159711201984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,16384,0.25983999570210775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,7168,0.11594453652699788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,8192,0.13131840229034425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,6144,0.10049813588460285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,10240,65536,1.346820322672526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3584,0.06070079803466797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,3072,0.05465813477834066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,4096,0.06807253360748292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,5120,0.08420586585998535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1536,0.02807786663373311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,1024,0.02031573255856832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2560,0.04511466821034749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,2048,0.037481598059336346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,512,0.013214932878812155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,768,0.016609066724777223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,256,0.009416533509890239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,128,0.007457066575686138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,64,0.006513066589832306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,32,0.0066655998428662615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,10240,0.14289706548055012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,12288,0.17122666041056317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,16384,0.2317311922709147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,8192,65536,1.0361109415690104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,8192,0.11530240376790364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,7168,0.10193173090616862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,6144,0.090338134765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3072,0.04702613353729248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,4096,0.06002666552861532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,5120,0.07635413010915121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,3584,0.05318719943364462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2048,0.03180053234100342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1536,0.02510400017102559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,1024,0.01802560091018677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,2560,0.039918935298919676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,768,0.01477013329664866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,512,0.011403733491897583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,256,0.008827733000119527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,128,0.007747200131416321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,64,0.007367466886838277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,32,0.007736533383528392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,10240,0.11902826627095539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,12288,0.14050025939941407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,16384,0.1912757396697998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,7168,65536,0.9044480005900064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,8192,0.09592426617940267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,7168,0.08388906319936117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,6144,0.0745685338973999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,5120,0.06237546602884928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,4096,0.05024213393529257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3072,0.03943893512090047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,3584,0.04408746560414632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2560,0.03291306694348653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,2048,0.02717546621958415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1536,0.021528534094492593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,1024,0.01595626672108968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,768,0.013769599795341491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,256,0.007701333363850911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,512,0.010679466525713603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,128,0.006491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,64,0.005831466615200042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,32,0.005992533266544342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,10240,0.10578347047170003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,12288,0.12345493634541829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,16384,0.16224533716837566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,6144,65536,0.7368661244710286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,8192,0.08563199838002523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,7168,0.07398826281229655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,6144,0.06330026785532633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,5120,0.054520531495412194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,4096,0.0447221318880717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3584,0.03891733487447103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,3072,0.03361173470815022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2560,0.02874559958775838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1536,0.019441066185633342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,2048,0.024190932512283325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,1024,0.014398933450380961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,768,0.012435199817021687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,512,0.010110933581988018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,256,0.007165866593519847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,128,0.006097066899140676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,64,0.005374933282534281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,32,0.005570133527119955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,10240,0.0828437328338623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,12288,0.09844586849212647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,16384,0.12938986619313558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,5120,65536,0.6474933624267578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,8192,0.06674346923828126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,7168,0.05897706747055054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,5120,0.0438538670539856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,6144,0.050538667043050135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3584,0.03105386694272359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,3072,0.02717120051383972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,4096,0.03515413204828898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2560,0.022860799233118692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,2048,0.018973867098490395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1536,0.015513599912325541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,1024,0.012066133817036946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,768,0.009892266988754273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,256,0.00619946668545405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,512,0.007842133442560833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,128,0.005310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,64,0.0047765334447224935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,32,0.0049333333969116214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,4096,65536,0.5229557355244954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,10240,0.07326719760894776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,12288,0.0859328031539917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,16384,0.11407252947489421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,5120,0.038335998853047684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,7168,0.052425599098205565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,8192,0.06141440073649088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,6144,0.0448309342066447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,4096,0.03140053351720174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3584,0.028013867139816285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,3072,0.024809600909550984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2560,0.02076586683591207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,2048,0.0178656001885732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1536,0.014738133549690247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,1024,0.011383466919263204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,768,0.009819733103116353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,512,0.008180266618728638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,256,0.006468266745408376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,128,0.0055061335364977515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,64,0.005064533154169718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,32,0.005392000079154968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,10240,0.06306666533152262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3584,65536,0.4441664059956868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,12288,0.07415359814961751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,16384,0.10080853303273518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,5120,0.0336736003557841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,7168,0.045393065611521406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,8192,0.050785064697265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,6144,0.03878186543782552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,4096,0.02765013376871745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3072,0.021291732788085938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,3584,0.023990400632222495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2560,0.018029866615931193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,2048,0.015154133240381876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1536,0.01262079974015554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,768,0.008021333316961924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,1024,0.009730133414268493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,512,0.006546133259932201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,256,0.005510400235652924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,128,0.0046623999873797095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,64,0.0042912001411120095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,32,0.004377600053946177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,3072,65536,0.3883530616760254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,12288,0.06474986473719278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,10240,0.05570880174636841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,16384,0.08477653662363688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,8192,0.044939732551574706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,6144,0.034687999884287515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,7168,0.038916265964508055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,5120,0.028542933861414592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,4096,0.023796266317367552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3584,0.021154133478800456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,3072,0.01821440060933431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2560,0.015761066476504007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,2048,0.013570132851600646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1536,0.011139200131098429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,1024,0.008781866232554118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,768,0.007272533575693766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,256,0.005028266708056132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,512,0.006266666452089946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,128,0.004403199752171834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,64,0.004206933577855428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,32,0.0042250668009122215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2560,65536,0.33584321339925133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,12288,0.07320853074391684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,10240,0.06157333453496298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,16384,0.09430186748504639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,8192,0.04878933429718017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,6144,0.03715519905090332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,5120,0.03214933276176453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,7168,0.0425439993540446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,4096,0.021177599827448525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3584,0.018525866667429607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,3072,0.01665173371632894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2560,0.014944000045458474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,2048,0.012492799758911132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1024,0.007201066613197327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,1536,0.010057600339253743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,512,0.005364266534646353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,768,0.006263466676076253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,128,0.003967999915281932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,256,0.004478933413823446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,64,0.0037450666228930154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,32,0.003790933390458425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,2048,65536,0.26386987368265785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,10240,0.0353493332862854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,12288,0.04218666553497315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,8192,0.029736532767613726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,16384,0.05829439957936605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,7168,0.02575040062268575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,5120,0.01890773375829061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,6144,0.022272000710169472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,4096,0.015461333592732749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3584,0.014004266262054444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,3072,0.012702932953834534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2560,0.011288533608118694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,2048,0.012570666273434958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1536,0.009781333804130554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,1024,0.007322666545708974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,768,0.006363733112812043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,512,0.0051018665234247845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,256,0.0042026668787002565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,128,0.003736533224582672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,64,0.003553066651026408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,32,0.0036629334092140196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1536,65536,0.2035967985788981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,12288,0.050247466564178465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,10240,0.04284799893697103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,8192,0.0335317333539327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,16384,0.041677868366241454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,7168,0.030713599920272828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,4096,0.013772799571355184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,6144,0.026743467648824053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,5120,0.017501866817474364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3584,0.012840533256530761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,3072,0.012076800068219502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2048,0.008237866560618083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,2560,0.01032533347606659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1536,0.007006933291753133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,1024,0.005509333312511444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,768,0.004964266717433929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,512,0.00429013321797053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,256,0.0037546666959921518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,128,0.0034101332227389016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,64,0.0032106667757034303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,32,0.0033290666838486993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,1024,65536,0.13863360087076823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,8192,0.01867626706759135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,10240,0.02335360050201416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,12288,0.02781760096549988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,16384,0.03444053332010905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,6144,0.014786133170127868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,7168,0.01697280009587606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,4096,0.01088853379090627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,5120,0.013090133666992188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2560,0.011236266295115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,2048,0.009471999605496724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3584,0.009841066598892213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,3072,0.00867733359336853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1024,0.005987200140953064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,1536,0.008041599889596303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,768,0.005293866495291392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,512,0.004444799820582072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,65536,0.11217706998189289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,256,0.003735466549793879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,128,0.003399466723203659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,64,0.0031914666295051576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,768,32,0.00314026673634847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,12288,0.021702400843302407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,8192,0.01626240015029907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,10240,0.019733333587646486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,16384,0.02762239972750346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,5120,0.011740799744923909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,7168,0.014922666549682616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,4096,0.009781333804130554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,6144,0.013222400347391763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3584,0.009063466389973959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,3072,0.008117333551247915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2560,0.0077002664407094315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,2048,0.006165333092212677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1536,0.005419733126958212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,1024,0.004634666442871094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,768,0.004273066421349844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,512,0.003788800040880839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,65536,0.07855786482493082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,256,0.0034133332471052804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,128,0.003180799881617228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,64,0.0029546665648619336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,512,32,0.0030720000465710956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,10240,0.014477866888046264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,8192,0.012547199924786886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,12288,0.015974400440851848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,16384,0.017349332571029663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,5120,0.009981866677602131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,7168,0.011752532919247945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,6144,0.011636267105738323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,4096,0.008524800340334576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2560,0.006664533416430156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3584,0.007857066889603932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,3072,0.007054933408896129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,2048,0.005819733440876007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1024,0.004403199752171834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,1536,0.005117866893609365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,512,0.0036778666079044344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,65536,0.05619200070699056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,768,0.004043733328580856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,256,0.003328000009059906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,128,0.0030826665461063385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,32,0.0029535998900731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,256,64,0.0028864001234372456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,10240,0.011113599936167399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,8192,0.009393067161242167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,12288,0.011745066444079081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,16384,0.011585066715876263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,6144,0.00841813286145528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,5120,0.008411733309427898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,7168,0.009146666526794434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,4096,0.007761066655317943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2560,0.006252799928188324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,2048,0.005525333185990652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3584,0.0075914666056633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,3072,0.006905599931875865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,65536,0.04663466612497966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1024,0.004175999760627746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,1536,0.00486826648314794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,768,0.0038005332152048744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,256,0.0030954666435718536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,512,0.0034613333642482757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,128,0.0030303999781608583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,64,0.002918400118748347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,128,32,0.0028149334092934927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,16384,0.012544000148773193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,8192,0.008472533027331034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,10240,0.009517866373062133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,12288,0.010319999853769938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,6144,0.007578666508197785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,4096,0.006684799989064534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,5120,0.007149866720040639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,7168,0.00811520020167033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2560,0.00618453323841095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3072,0.006776533524195353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,3584,0.007177599767843883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,2048,0.005480533341566721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,65536,0.042172801494598386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1536,0.0047978664437929785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,1024,0.004089600096146265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,768,0.00382080003619194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,512,0.0034613333642482757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,256,0.0031093334158261614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,128,0.0030229332546393077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,64,0.0028234665592511495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,64,32,0.002828799933195114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,10240,0.007959466675917308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,16384,0.009514666597048442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,12288,0.008459732929865519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,8192,0.007283199826876323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,5120,0.0069482664267222095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,6144,0.006525866687297821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,4096,0.006524799764156342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,7168,0.006778666873772939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2560,0.006087466577688853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3072,0.0066442668437957765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,2048,0.0054058666030565895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,3584,0.006932266553243001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,65536,0.040193065007527666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1536,0.004759466648101807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,768,0.0037119999527931214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,512,0.0034080001215140024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,1024,0.004054400076468786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,256,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,64,0.0028010666370391845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,128,0.002881066749493281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,768,32,32,0.0027402666707833606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,7168,0.6100063959757487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,8192,0.6948853174845377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,10240,0.8775103886922201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,12288,1.034110959370931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,6144,0.5249984105428059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,5120,0.44015146891276047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,4096,0.3581589380900065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,16384,1.403936004638672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3584,0.3201066652933756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2560,0.23299733797709146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,3072,0.27037226359049477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,2048,0.1874293327331543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1024,0.10136213302612304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,1536,0.1510954697926839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,64,0.03162239988644917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,128,0.03624000151952107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,256,0.04461866617202759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,768,0.08180906772613525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,512,0.06412373383839926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,65536,32,0.036082132657368975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,8192,0.17898240089416503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,10240,0.21733013788859049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,12288,0.2598624070485433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,16384,0.34650986989339194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,7168,0.1569152037302653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,6144,0.1328821341196696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,5120,0.11151573657989503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,4096,0.09179413318634033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3072,0.07001067002614339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,3584,0.08139093716939291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2560,0.05886186758677164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,768,0.02257173260052999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1024,0.027804799874623615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,1536,0.03794133265813192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,2048,0.048749868075052896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,512,0.016761600971221924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,256,0.012032000223795573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,128,0.009435733159383137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,64,0.007621333499749501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,32,0.008162133395671844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,10240,0.16858560244242352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,12288,0.2034623940785726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,16384,0.2687157313028971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,7168,0.12057920296986897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,8192,0.13549013137817384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,6144,0.10360213120778401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,4096,0.07029653390248616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,5120,0.08746346632639566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3584,0.06232959826787313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2560,0.04613226652145386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,2048,0.03768746852874756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,3072,0.05379626750946045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1024,0.02113386591275533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,768,0.016797866423924765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,1536,0.02954559922218323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,512,0.01313920021057129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,256,0.009563733140627544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,128,0.007241599758466085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,64,0.006455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,32,0.006617600222428639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,16384,65536,1.3765951792399087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,12288,0.16482027371724445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,10240,0.14267627398173016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,16384,0.21796266237894693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,12288,65536,1.0972789764404296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,7168,0.09806079864501953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,6144,0.08550613721211751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,8192,0.11155200004577637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3072,0.04549653530120849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,3584,0.052108800411224364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,4096,0.05825813213984171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,5120,0.07654613653818766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1536,0.02627519965171814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,1024,0.01895786722501119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2048,0.032228267192840575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,2560,0.03886293172836304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,768,0.01508799990018209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,512,0.012074666221936543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,256,0.009033600489298504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,128,0.006958933174610138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,64,0.006171733140945435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,32,0.0064522668719291685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,10240,0.1118016004562378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,12288,0.12962559858957928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,16384,0.1720597267150879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,8192,0.09065706729888916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,7168,0.07752426465352377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,6144,0.06725760300954184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,10240,65536,0.8716192245483398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,5120,0.05795093377431234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,4096,0.04602453311284383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3584,0.041206399599711105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,3072,0.03767893314361572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1024,0.015019733707110086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,1536,0.020281600952148437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2048,0.025614933172861738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,2560,0.031146667400995892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,768,0.012425600488980611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,512,0.01002346674601237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,128,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,256,0.007347199817498524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,64,0.005371733506520589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,32,0.005571199953556061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,10240,0.10181439717610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,12288,0.12258986632029216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,16384,0.1601696014404297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,8192,65536,0.6837685267130534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,7168,0.07346346378326415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,8192,0.08229973316192626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,6144,0.06016319990158081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,5120,0.05573973258336386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,4096,0.041750399271647136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3584,0.036424533526102705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,3072,0.031396265824635824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2560,0.02749333381652832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,2048,0.02219946583112081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1536,0.01728640000025431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,1024,0.012615467111269632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,768,0.010660266876220703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,512,0.008318933347860973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,256,0.006117333471775055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,128,0.005356800059477488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,64,0.005041066805521647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,32,0.005182933310667674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,10240,0.08568639755249023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,12288,0.10167360305786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,16384,0.133622407913208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,7168,65536,0.6489493052164714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,8192,0.06909226576487223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,7168,0.06056106487909953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,6144,0.0522325317064921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,5120,0.04599893490473429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,4096,0.0360479990641276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3584,0.032015999158223465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2560,0.024289067586263022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,3072,0.027845333019892376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1536,0.01580693324406942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,2048,0.019756799936294554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,1024,0.012033067146937053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,768,0.010258133212725323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,512,0.008497066299120585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,256,0.006116266548633576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,128,0.005261866748332978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,32,0.004990933338801066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,64,0.004810666541258494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,10240,0.07241706848144532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,12288,0.08662613232930502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,16384,0.11602239608764649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,6144,65536,0.5253653208414714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,6144,0.04583573341369629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,7168,0.051780267556508386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,8192,0.05833813349405924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,5120,0.0394378662109375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3072,0.02444266676902771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,4096,0.031914667288462324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,3584,0.02792853315671285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2560,0.021155200401941934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,2048,0.01769066651662191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1024,0.011441066861152649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,768,0.010130133231480916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,1536,0.01446613371372223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,256,0.006291200220584869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,128,0.005155199766159057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,64,0.004657066861788432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,512,0.007964799801508587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,32,0.004845866560935974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,12288,0.06708587010701497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,10240,0.05652159849802653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,5120,65536,0.45119466781616213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,16384,0.09064640204111735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,6144,0.03687146504720052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,5120,0.03043839931488037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,7168,0.04047893285751343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,8192,0.04558826684951782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3584,0.022188800573349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,3072,0.01898026665051778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2560,0.016203733285268147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,4096,0.024565333127975465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,2048,0.013788800438245138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1536,0.01160533328851064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,1024,0.009051733215649923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,768,0.007433600227038066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,512,0.006178133189678192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,256,0.0050911997755368555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,64,0.003999999910593033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,128,0.004468266665935516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,32,0.004198400179545084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,4096,65536,0.3514591852823893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,10240,0.05153173208236694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,12288,0.05936959981918335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,16384,0.07912000020345053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,7168,0.036663464705149335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,5120,0.026817067464192705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,8192,0.04158186515172323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,6144,0.031144533554712934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3584,0.019349332650502524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,3072,0.016876800855000814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,4096,0.022175999482472737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2560,0.014663466811180114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,2048,0.012337066729863485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1536,0.010440533359845478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,768,0.00689279983441035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,1024,0.008289066453774769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,512,0.005721599857012431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,256,0.00476800004641215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,64,0.0038442666331926978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,128,0.0042367999752362575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,32,0.003962666789690653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3584,65536,0.30573012034098307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,12288,0.051874132951100674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,10240,0.047492265701293945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,16384,0.07038293679555258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,8192,0.03710399866104126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,5120,0.023964800437291465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,6144,0.027635200818379717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,7168,0.031549866994222006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,4096,0.01914026737213135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3584,0.01725333333015442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,3072,0.014885333180427552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2560,0.013006933530171714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,2048,0.011400533715883891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1024,0.007202133536338806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,1536,0.009477333227793375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,512,0.005412266651789347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,768,0.0062837332487106325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,64,0.003701333453257879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,256,0.004499199986457825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,128,0.004043733328580856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,32,0.0038506666819254553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,3072,65536,0.2744640032450358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,12288,0.047270401318868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,10240,0.04984426498413086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,16384,0.05888319810231527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,8192,0.03240533272425334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,6144,0.02836266756057739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,5120,0.024563199281692503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,7168,0.03263466755549113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,4096,0.016331733266512553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3072,0.015337600310643514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2560,0.01306880017121633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,3584,0.01679146687189738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,2048,0.010875733693440755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1536,0.008721066514650981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,1024,0.0066101332505544026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,768,0.005955199897289276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,512,0.005037866532802582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,256,0.004267733295758565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,128,0.003868799904982249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,64,0.0035690667728583017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,32,0.003731200098991394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2560,65536,0.2312170664469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,16384,0.053236265977223717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,12288,0.04987093210220337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,10240,0.04415466785430908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,8192,0.03397653500239055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,5120,0.022370133797327676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,7168,0.030522666374842328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,6144,0.025038933753967284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,4096,0.017498666048049928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2560,0.012331733107566833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3584,0.015927466750144958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,3072,0.014320000012715658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,2048,0.011174399654070537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1024,0.006358399987220764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,512,0.0047989333669344585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,1536,0.008571733037630718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,768,0.005783466498057047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,128,0.003656533360481262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,256,0.004038399954636892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,64,0.0034442665676275887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,32,0.0034485332667827605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,2048,65536,0.18289492925008138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,10240,0.03824533224105835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,8192,0.027767467498779296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,12288,0.04016000032424927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,16384,0.04146346648534139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,7168,0.02569813330968221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,6144,0.02145493427912394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,5120,0.019966934124628702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,4096,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3584,0.013762133320172629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,3072,0.012403200070063274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2560,0.010441600282986959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,2048,0.009136000275611877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1024,0.00551146666208903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,1536,0.007652266820271809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,768,0.004884266853332519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,512,0.004223999877770742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,256,0.0036757332583268487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,128,0.0033088001112143196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,64,0.003151999910672506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,32,0.003270400067170461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1536,65536,0.1403861363728841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,8192,0.02105493346850077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,12288,0.028701867659886676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,10240,0.025600000222524004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,16384,0.03165973424911499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,7168,0.01829973260561625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,6144,0.016914133230845133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,4096,0.012317867080370585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,5120,0.013522133231163025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3584,0.010749866565068562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2560,0.008026666442553202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,3072,0.009954133629798889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,2048,0.006822399795055389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1536,0.006011733412742614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,768,0.004343466460704803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,1024,0.004869333406289419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,512,0.003921066721280416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,65536,0.09758293628692627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,256,0.003521066655715307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,64,0.0030933332939942675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,128,0.0032629333436489105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,1024,32,0.0031946666538715364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,12288,0.025135999917984007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,10240,0.02295253276824951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,8192,0.016356266538302102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,16384,0.026449066400527955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,7168,0.014980266491572062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,5120,0.013553067048390707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,4096,0.011036800344785054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,6144,0.013264000415802002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2560,0.007880533238252004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3584,0.009273599584897358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,3072,0.008450133601824443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,2048,0.007157333195209503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,65536,0.07692373593648275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1536,0.005947733422120413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,1024,0.004628266890843709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,768,0.004183466732501984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,512,0.003751466671625773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,256,0.0034933333595593774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,64,0.0029834667841593427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,128,0.00310506671667099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,768,32,0.003049599876006444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,10240,0.018734933932622273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,12288,0.01996586720148722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,16384,0.020181334018707274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,8192,0.01343786617120107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,7168,0.014057599504788718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,6144,0.013717333475748697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,5120,0.012686933080355326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,4096,0.0092031995455424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3072,0.007919999957084655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2560,0.007075199981530507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,3584,0.00869546631971995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,2048,0.006362666686375936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,65536,0.05776853164037069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1536,0.005559466779232025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,768,0.004250666499137879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,1024,0.0046079998215039575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,512,0.0038624001046021783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,256,0.003416533271471659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,128,0.0030720000465710956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,64,0.002963199963172277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,512,32,0.0028789333999156954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,10240,0.0116565336783727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,8192,0.008753066261609394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,12288,0.012050132950146992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,16384,0.01341973344484965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,6144,0.008709333340326945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,7168,0.010934399565060933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,5120,0.009213866790135701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,4096,0.008218666911125183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,65536,0.042218665281931564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3584,0.007699200014273326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,3072,0.007643733421961467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2560,0.006324266890684764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,2048,0.005601066847642263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1536,0.004937600096066793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,1024,0.004273066421349844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,768,0.0038634667793909705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,512,0.0035466666022936503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,128,0.0030154667794704436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,256,0.003205333401759466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,64,0.002868266652027766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,256,32,0.002882133424282074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,8192,0.008472533027331034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,12288,0.009324799974759419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,16384,0.01099733312924703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,10240,0.01015786627928416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,7168,0.008221866687138875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,4096,0.007512533167997997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,5120,0.008239999910195668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,6144,0.008867200215657551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,65536,0.034302934010823564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3072,0.006874666611353557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,3584,0.007206400235493978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2560,0.006234666705131531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,2048,0.0055402666330337524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1536,0.0048320000370343525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,1024,0.004147200038035711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,768,0.0037834666669368743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,512,0.0034730667869249977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,256,0.0032416000962257386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,128,0.0028885332246621448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,64,0.0028480000793933867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,128,32,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,12288,0.00819946676492691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,8192,0.0070933332045873005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,10240,0.0076447998483975725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,16384,0.009506133198738099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,6144,0.0065984000762303666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,7168,0.006878933310508728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,5120,0.007000533243020375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,4096,0.006579199930032094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,65536,0.03158826629320781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2560,0.006050133208433787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3584,0.007010133564472198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,3072,0.006702933212121327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,2048,0.005368533233801523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1536,0.004711466530958811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,1024,0.00405973345041275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,512,0.0034143999218940735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,768,0.003913599997758865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,128,0.0028917332490285236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,256,0.0032000000278155005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,64,0.002734933296839396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,64,32,0.0027850667635599775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,10240,0.006950399776299794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,16384,0.007897600034872691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,8192,0.006822399795055389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,12288,0.007221333185831706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,7168,0.006712533533573151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,6144,0.006435200075308482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,4096,0.006433066725730896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,5120,0.006875733534495036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3584,0.006922666728496551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2048,0.005378133555253347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,2560,0.006066133578618368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,3072,0.006580266853173573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1536,0.00470719983180364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,1024,0.0040554667512575785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,768,0.003702399879693985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,512,0.003386666625738144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,65536,0.029313067595163982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,256,0.0030730667213598887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,128,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,64,0.0027818667391935987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,512,32,32,0.0027744000156720476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,7168,0.4749279975891113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,8192,0.5385386784871419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,10240,0.6923850377400715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,12288,0.8344074885050455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,5120,0.34293012619018554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,6144,0.40946772893269856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,4096,0.2868586540222168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,16384,1.0801748911539713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,3584,0.26092373530069984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,2048,0.15009387334187824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,2560,0.18167999585469563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,3072,0.21410560607910156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,1024,0.08250239690144857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,256,0.0418229341506958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,768,0.06731839974721274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,512,0.05738879839579264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,128,0.030019199848175047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,64,0.02648640076319377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,1536,0.12327466805775959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,65536,32,0.025795199473698932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,8192,0.1414122740427653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,10240,0.17107200622558594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,12288,0.20377813975016273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,16384,0.272985585530599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,7168,0.12445440292358398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,6144,0.10470506350199382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,5120,0.08796266714731851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,4096,0.07393173376719156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,3584,0.06461013158162435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,3072,0.05477973222732544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,2560,0.04662719964981079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,1536,0.031124265988667806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,2048,0.03865706523259481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,768,0.01734506686528524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,1024,0.021541333198547362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,512,0.013378133376439413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,256,0.009913600484530131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,64,0.00658133327960968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,128,0.00761599987745285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,32,0.006764799853165944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,10240,0.12746559778849284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,12288,0.14845973650614422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,16384,0.1964245319366455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,7168,0.08871359825134277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,8192,0.10293866793314617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,6144,0.07714560031890869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,4096,0.05375146468480428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,5120,0.06470719973246256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,3584,0.0468725323677063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,3072,0.04169493516286214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,2560,0.03494079907735188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,2048,0.030907734235127764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,1024,0.016557866334915163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,1536,0.023592533667882283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,512,0.011084799965222675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,768,0.013953066865603127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,256,0.00844053328037262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,128,0.006614399949709575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,64,0.005859200159708659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,32,0.006026666859785716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,16384,65536,1.0747210184733071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,12288,0.13355414072672528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,10240,0.11545600096384685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,16384,0.17120320002237957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,8192,0.09022719860076904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,12288,65536,0.7819957097371419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,6144,0.06818666458129882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,7168,0.08131946722666422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,3072,0.03662933508555095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,5120,0.05840213298797607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,4096,0.049728000164031984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,3584,0.04215466578801473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,1536,0.0210207998752594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,1024,0.014909866452217101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,2048,0.02630400061607361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,2560,0.031541333595911665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,768,0.012777599692344665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,512,0.010290132959683736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,256,0.007414400080839793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,128,0.006166400015354156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,64,0.005449600021044413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,32,0.005666133264700572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,10240,0.08692586421966553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,12288,0.10383893648783367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,16384,0.1356170654296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,7168,0.061970134576161705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,8192,0.07001386483510336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,10240,65536,0.6710570653279622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,6144,0.05324800014495849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,4096,0.0367082675298055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,3584,0.03452800114949544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,3072,0.02874666651089986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,5120,0.04524906476338704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,1024,0.012045866250991822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,1536,0.01604693333307902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,2048,0.021282132466634116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,2560,0.024634667237599692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,768,0.01037440001964569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,512,0.008516266942024231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,256,0.006222933530807495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,128,0.00528106689453125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,64,0.0047872001926104225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,32,0.004996266464392344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,10240,0.07610346476236979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,12288,0.09157546361287436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,16384,0.1206549326578776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,8192,65536,0.5509834925333659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,7168,0.05526080131530762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,8192,0.062080001831054686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,6144,0.04798613389333089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,5120,0.04387306769688924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,3584,0.029882667462031047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,4096,0.03320639928181966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,3072,0.02656000057856242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,2560,0.023141332467397056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,1536,0.015000533064206442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,2048,0.01864746610323588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,1024,0.011690666278203327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,768,0.010230400164922078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,512,0.008561066786448161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,128,0.005721599857012431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,256,0.006491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,64,0.005188266436258951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,32,0.005427200098832448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,10240,0.06908373037974039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,12288,0.08256853421529134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,16384,0.10219626426696778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,7168,65536,0.46050453186035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,7168,0.04776000181833903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,8192,0.05320426623026529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,6144,0.04056853453318278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,5120,0.036824532349904376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,3584,0.02553279995918274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,3072,0.023137066761652628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,4096,0.028806400299072266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,2560,0.019949867328008016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,2048,0.015780267119407655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,1024,0.010154666503270467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,1536,0.01316266655921936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,768,0.008730666836102803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,512,0.0072405333320299785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,256,0.005443199972311656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,128,0.004749866823355356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,64,0.004279466470082601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,32,0.004473599791526795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,10240,0.0571616013844808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,12288,0.06719146569569906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,16384,0.08866559664408366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,6144,65536,0.40430186589558914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,8192,0.046350932121276854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,7168,0.04098133246103923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,6144,0.035631998380025225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,5120,0.03345280090967814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,4096,0.025167999664942424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,3072,0.01984213391939799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,2560,0.016457600394884746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,3584,0.022779732942581177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,2048,0.013798399766286214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,1536,0.01167146662871043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,1024,0.008987733721733093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,768,0.008026666442553202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,256,0.005154133339722951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,512,0.006594133377075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,128,0.004397866626580557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,64,0.004251733422279358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,32,0.004364799956480662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,5120,65536,0.36259733835856117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,12288,0.07466560204823812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,10240,0.06688960393269858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,16384,0.097325865427653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,6144,0.03998186588287354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,8192,0.04159253438313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,7168,0.04580693244934082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,5120,0.03406293392181396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,3584,0.018574933211008705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,2560,0.014325333635012307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,3072,0.016687999169031777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,4096,0.02046826680501302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,1536,0.010116266210873921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,1024,0.0077344000339508055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,2048,0.012180266777674358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,768,0.006548266609509785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,512,0.005355733136336008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,256,0.0044714664419492085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,128,0.003934933245182038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,64,0.0036544000109036768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,32,0.0038549333810806276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,4096,65536,0.2754549344380697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,12288,0.051054934660593666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,10240,0.044343467553456625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,16384,0.06837226549784342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,8192,0.0379530668258667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,7168,0.03136853377024333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,6144,0.027483733495076497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,5120,0.023770666122436522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,4096,0.021663999557495116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,2560,0.013980799913406372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,3584,0.01700053413709005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,3072,0.01580586632092794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,2048,0.012038399775822956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,1536,0.011478400230407715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,1024,0.008920533458391826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,768,0.007177599767843883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,512,0.005827199916044871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,256,0.004657066861788432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,128,0.004093866546948751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,64,0.0037237333754698435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,32,0.004005333284536997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3584,65536,0.24404800732930504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,12288,0.0439242680867513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,10240,0.04214506546656291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,16384,0.05736426512400309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,8192,0.03350720008214315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,6144,0.025382399559020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,5120,0.0204202671845754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,7168,0.02805440028508504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,4096,0.018245333433151247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,3584,0.014662399888038635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,3072,0.013385599851608277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,2560,0.011662933230400085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,2048,0.009872000416119893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,1536,0.009127466877301534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,1024,0.006682666639486949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,768,0.0058677335580190025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,512,0.0051125332713127134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,256,0.004171733558177948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,128,0.003685333331425985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,64,0.0035125332574049628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,32,0.003692800054947535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,3072,65536,0.21277227401733398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,12288,0.039879465103149415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,10240,0.03390080134073893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,8192,0.03009066581726074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,16384,0.05384320020675659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,5120,0.018564265966415406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,4096,0.014245333274205527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,6144,0.021901865800221763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,7168,0.02510720094045003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,3584,0.014221866925557455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,2560,0.010913067062695821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,3072,0.011490133404731751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,2048,0.009598933657010396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,1536,0.007527466615041096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,768,0.005262933174769084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,1024,0.005845333139101664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,512,0.004633600016434988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,256,0.003933866570393244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,32,0.0034474665919939675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,64,0.003370666752258936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,128,0.0035743998984495797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2560,65536,0.18295146624247233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,10240,0.040491731961568196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,8192,0.031666133801142374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,12288,0.047654398282368976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,16384,0.0488042672475179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,6144,0.024421334266662598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,5120,0.020601600408554077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,7168,0.029041065772374468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,4096,0.014805333813031516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,3584,0.012946133812268576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,3072,0.011870933572451274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,2560,0.01030399998029073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,2048,0.00897173285484314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,1536,0.007375999788443248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,1024,0.005610666672388713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,768,0.00489386667807897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,512,0.004347733159859975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,256,0.003735466549793879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,128,0.003420799970626831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,65536,0.14803199768066405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,64,0.0032799998919169106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,2048,32,0.003320533285538355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,8192,0.020626133680343627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,10240,0.02615999976793925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,12288,0.030920533339182536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,16384,0.036524800459543864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,7168,0.018542933464050292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,6144,0.01651093363761902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,5120,0.014083199699719749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,4096,0.012077866991360982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,3584,0.010558933019638062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,2560,0.01000853379567464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,3072,0.009843200445175171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,2048,0.008566400408744812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,1536,0.007156266768773396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,1024,0.005409066875775656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,768,0.004818133513132731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,65536,0.11343999703725179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,512,0.004281599819660187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,256,0.0036490666369597114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,128,0.003366400053103765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,64,0.00322026660044988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1536,32,0.0030986666679382324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,12288,0.027727999289830524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,10240,0.02528426647186279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,8192,0.016360533237457276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,16384,0.029366399844487506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,7168,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,5120,0.011823999881744384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,4096,0.009993599851926167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,6144,0.013398399949073792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,3584,0.009406933188438415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,3072,0.008408533533414204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,2560,0.00869546631971995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,2048,0.006841599941253662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,1536,0.0058442667126655575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,65536,0.10195199648539226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,1024,0.004669866462548574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,768,0.004344533383846283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,512,0.0038250667353471124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,256,0.0034122665723164878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,128,0.0031413334111372627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,64,0.0030165334542592366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,1024,32,0.003068800022204717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,16384,0.0250709335009257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,12288,0.020747733116149903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,10240,0.019336533546447755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,8192,0.01620266636212667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,7168,0.014667733510335287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,6144,0.01299626628557841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,5120,0.011505066355069479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,4096,0.009848533074061076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,3584,0.00914026697476705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,2560,0.007269333302974701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,3072,0.008189866443475087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,2048,0.006307200094064077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,65536,0.06539733409881592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,1536,0.00556160012880961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,1024,0.004567466676235199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,768,0.004217599829037985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,512,0.003765333443880081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,256,0.0034229333202044168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,64,0.002997333308060964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,128,0.003035733352104823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,768,32,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,16384,0.017248000701268515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,12288,0.01588053305943807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,10240,0.01437333325544993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,8192,0.01258666714032491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,7168,0.011920000116030376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,6144,0.012056533495585125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,5120,0.010494933525721232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,4096,0.008599467078844706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,3584,0.008140799899895985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,65536,0.05182400147120157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,3072,0.007106133302052816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,2560,0.007743999858697255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,2048,0.006505600114663441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,1536,0.0055285334587097164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,768,0.004262400170167288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,1024,0.004695466657479604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,512,0.003739733248949051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,256,0.0033845332761605583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,128,0.003032533327738444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,64,0.0029088000456492106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,512,32,0.002942933390537898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,10240,0.011327999830245971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,16384,0.011620266238848369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,12288,0.012408533692359924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,8192,0.010076799988746643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,5120,0.008449066678682964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,7168,0.009282132983207703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,6144,0.008593066533406576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,4096,0.00768746683994929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,65536,0.03692266543706258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,3072,0.006884266436100006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,3584,0.0076223999261856076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,2560,0.006222933530807495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,2048,0.005538133283456167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,1536,0.004851200183232625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,1024,0.004167466859022776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,768,0.0038773333032925926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,512,0.0034474665919939675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,256,0.003123199939727783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,128,0.002898133297761281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,64,0.0028362666567166646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,256,32,0.0029088000456492106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,12288,0.010566400488217671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,16384,0.013105066617329917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,10240,0.009492266178131103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,8192,0.008498133222262064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,7168,0.00803413341442744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,6144,0.007521066566308339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,5120,0.007088000078996022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,4096,0.0067114666104316715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,65536,0.029198932647705077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,3584,0.007152000069618225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,3072,0.006840533514817555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,2560,0.006197333335876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,2048,0.005549866457780202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,1536,0.004837333162625631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,1024,0.0041354666153589886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,512,0.0034400001168251038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,768,0.0037845333417256674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,256,0.003159466634194056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,128,0.0029781334102153777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,64,0.002855466554562251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,128,32,0.0028714666763941447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,16384,0.010573866963386535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,8192,0.007807999849319458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,10240,0.008340266346931458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,12288,0.009051733215649923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,6144,0.006563200056552887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,7168,0.00693333347638448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,5120,0.006923733154932658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,4096,0.006611200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,65536,0.026013867060343428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,3584,0.0070271998643875126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,3072,0.006662400066852569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,2560,0.0060576001803080235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,1536,0.004694400231043497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,2048,0.005378133555253347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,1024,0.0040448000033696495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,768,0.003701333453257879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,512,0.0033557333052158357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,256,0.0030623999734719592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,128,0.0029237332443396253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,64,0.002773333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,64,32,0.0027935999135176343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,12288,0.007929599781831106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,16384,0.00853653351465861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,10240,0.007230933507283528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,8192,0.00695253312587738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,4096,0.006393600006898243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,5120,0.006854400038719177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,7168,0.006647466619809468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,6144,0.006431999802589417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,65536,0.024449066321055094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,3072,0.006540800134340922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,3584,0.0069248000780741375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,2048,0.005342933535575867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,1536,0.004721066852410635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,2560,0.0060138667623202005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,1024,0.004053333401679992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,768,0.003705599904060364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,512,0.0033226666351159418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,128,0.002883200099070867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,256,0.003031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,64,0.0027647999425729113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,384,32,32,0.0027146667242050173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,7168,0.35365972518920896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,8192,0.397926394144694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,10240,0.4986240069071452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,12288,0.5963242848714192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,5120,0.2586197376251221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,6144,0.3067008018493652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,4096,0.2214346726735433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3584,0.20983786582946778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,16384,0.7695007960001627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2048,0.11637439727783203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,2560,0.14369386037190754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,3072,0.17079146703084308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,768,0.0525322675704956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1536,0.09155733585357666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,1024,0.07137813568115234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,512,0.04564799865086873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,256,0.032986666758855185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,128,0.021713066101074218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,32,0.016871466239293417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,65536,64,0.01625279982884725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,8192,0.10219199657440185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,10240,0.12668266296386718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,12288,0.1536853313446045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,16384,0.19711467425028484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,7168,0.09131946563720703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,5120,0.06757866541544597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,6144,0.08027413686116537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,4096,0.06296426852544149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3584,0.04978986581166585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2560,0.03786773284276326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,3072,0.043970131874084474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,768,0.014247467120488485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,2048,0.031575467189153036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1024,0.017978666226069133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,1536,0.025613866249720257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,512,0.010833066701889039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,256,0.00844373305638631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,64,0.005508266886075338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,128,0.006167466441790262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,32,0.005757866799831391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,10240,0.09562986691792806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,12288,0.1139690637588501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,16384,0.14621119499206542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,8192,0.07765866915384928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,7168,0.06813759803771972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,6144,0.05984106858571371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,5120,0.05129599968592326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,4096,0.042003198464711504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3584,0.037843199570973714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,3072,0.03344959815343221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2560,0.02900586724281311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,2048,0.02405973275502523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1024,0.013340800007184347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,768,0.011061333616574605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,1536,0.019266132513682047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,128,0.005167999863624572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,256,0.0064298664530118305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,512,0.009063466389973959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,64,0.004625066618124644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,32,0.005064533154169718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,16384,65536,0.7682538350423177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,10240,0.09227946599324545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,12288,0.09713706970214844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,16384,0.12616639931996662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,12288,65536,0.5733034769694011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,7168,0.059064535299936924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,8192,0.06668799718221029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,6144,0.052078934510548916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3072,0.029590400060017903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,4096,0.03692906697591146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,3584,0.03713066577911377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,5120,0.050012799104054775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1536,0.01686613361040751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,1024,0.013335466384887695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2560,0.025671466191609697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,2048,0.02131519913673401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,512,0.00897813340028127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,768,0.010745599865913391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,256,0.006542933483918508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,128,0.005106133222579956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,64,0.004620799918969473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,32,0.004939733445644379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,10240,0.06546239852905274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,12288,0.07831679979960124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,16384,0.1001792033513387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,8192,0.05339306592941284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,7168,0.047566934426625566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,6144,0.04214719931284587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,10240,65536,0.5286911964416504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,5120,0.036006398995717365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,4096,0.029734400908152263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3072,0.026765867074330645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,3584,0.026829866568247478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1536,0.013214932878812155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,1024,0.009774933258692425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2560,0.020536533991495767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,2048,0.016681599617004394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,768,0.00853760043780009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,512,0.007066666583220164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,256,0.005156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,128,0.004587733248869578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,64,0.00409706657131513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,32,0.004379733403523763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,10240,0.059704534212748205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,12288,0.0685482660929362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,16384,0.08846933046976725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,8192,65536,0.40501972834269206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,8192,0.046794664859771726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,7168,0.04203626712163289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,6144,0.03735893170038859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,5120,0.03623040119806926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3584,0.023796266317367552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,4096,0.026578134298324584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,3072,0.021336533625920615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2560,0.018496000766754152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1024,0.009001599748929341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,2048,0.01497706671555837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,768,0.007884799937407176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,1536,0.011528533697128297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,512,0.006387199958165486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,128,0.004187733431657155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,256,0.004828799764315287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,64,0.0038453333079814913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,32,0.004172799984614054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,10240,0.0584394653638204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,12288,0.06102933486302694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,16384,0.0789685328801473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,7168,65536,0.3333386739095052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,8192,0.04152959982554118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,7168,0.03746453523635864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,6144,0.03359466791152954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,5120,0.03251519997914632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,4096,0.023320533831914268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2560,0.016035200158754984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3584,0.021116799116134642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,3072,0.019236266613006592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1536,0.010640000303586323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,1024,0.008201600114504496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,2048,0.013182933131853739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,768,0.007274666428565979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,512,0.0056981335083643595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,256,0.004589866598447164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,128,0.004098133246103922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,64,0.0037503999968369803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,32,0.004005333284536997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,12288,0.05883306662241618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,10240,0.05119040012359619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,6144,65536,0.30537919998168944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,16384,0.06815360387166342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,7168,0.03473600149154663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,6144,0.03257386684417725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,5120,0.02744426727294922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,8192,0.039111467202504475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3072,0.01748266617457072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2560,0.014711466431617738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,3584,0.019452800353368126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,4096,0.0213045338789622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,2048,0.012052266796429952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1536,0.010420266787211101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,1024,0.007978666822115581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,768,0.006691200037797292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,512,0.005295999844868978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,256,0.004412800073623657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,128,0.0038901334007581077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,64,0.0036042665441830954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,32,0.003802666564782461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,10240,0.04331839879353841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,5120,65536,0.27779947916666664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,12288,0.05243733326594034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,16384,0.061845334370930996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,5120,0.024116265773773193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,8192,0.03186560074488322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,6144,0.0284223993619283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,7168,0.03330986698468526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2560,0.01230400005976359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3584,0.01616426706314087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,3072,0.015545599659283958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,4096,0.018589866161346436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,2048,0.01088853379090627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1536,0.008959999680519104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,768,0.0059456000725428265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,1024,0.007421866556008657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,512,0.0047082667549451195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,128,0.0035829332967599234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,256,0.004057600100835165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,64,0.0033546666304270422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,32,0.0035957333942254386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,4096,65536,0.21505173047383627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,12288,0.045950933297475176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,10240,0.03588053385416667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,16384,0.05536426703135172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,8192,0.03336639801661174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,7168,0.02650880018870036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,6144,0.023430399099985757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,5120,0.020124799013137816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,4096,0.016795732577641807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3584,0.015784533818562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2560,0.011772800485293071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,3072,0.014009599884351095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,2048,0.010137599706649781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1536,0.008423466483751934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,1024,0.00644053320089976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,768,0.005496533215045929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,512,0.00448639988899231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,256,0.003852800031503042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,128,0.003454933315515518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,64,0.0032373333970705668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,32,0.003387733300526937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3584,65536,0.18835946718851726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,12288,0.041048534711201984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,10240,0.036396801471710205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,16384,0.04963093201319377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,8192,0.029255467653274535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,4096,0.014988799889882406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,5120,0.019771732886632285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,6144,0.022363734245300294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,7168,0.025099732478459674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3584,0.013931733369827271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,3072,0.012436266740163167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2560,0.011147733529408772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,2048,0.009301333626111349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1536,0.00783679982026418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,768,0.004952533543109894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,1024,0.005975466469923655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,256,0.003681066632270813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,512,0.00420053352912267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,128,0.0033600000043710076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,32,0.003316266586383184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,64,0.0031658666829268134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,3072,65536,0.16058133443196615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,8192,0.025727999210357667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,12288,0.03603413502375285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,10240,0.032979200283686325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,16384,0.043277867635091144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,6144,0.017853866020838417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,4096,0.014140799641609192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,5120,0.01836479902267456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,7168,0.020529067516326903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3584,0.012737066547075907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,3072,0.011426132917404175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2560,0.010021332899729412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,2048,0.00857919951279958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1536,0.007229866584142049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,1024,0.00528959979613622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,768,0.004603733122348785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,512,0.004065066576004028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,256,0.00352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,65536,0.13924907048543295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,32,0.0032543999453385672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,128,0.0032479998966058097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2560,64,0.003102933367093404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,8192,0.0184661328792572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,10240,0.03183573285738627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,12288,0.03480106592178345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,16384,0.03768959840138753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,6144,0.01853546698888143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,5120,0.016646400094032288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,7168,0.019504000743230186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,4096,0.010818133751551311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2048,0.007075199981530507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3072,0.009107200304667155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,3584,0.010981333255767823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,2560,0.008310399949550629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1536,0.006233599781990051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,768,0.004489600161711375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,1024,0.004889599978923798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,512,0.003941333293914795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,65536,0.11404266357421874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,256,0.0035071998834609987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,128,0.0032543999453385672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,64,0.003101866692304611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,2048,32,0.0032960000137488045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,8192,0.017066667477289833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,10240,0.024103466669718424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,12288,0.027270400524139406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,16384,0.03036053379376729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,7168,0.015375999609629312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,6144,0.013544533650080362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,5120,0.014781866470972696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,4096,0.010696533322334289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3584,0.009634133179982502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2560,0.007952000200748443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,3072,0.00890880028406779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,2048,0.006877866884072621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,65536,0.0898751974105835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1024,0.004948266843954722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,1536,0.00589333325624466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,512,0.003718400001525879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,768,0.004163199911514918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,256,0.0033290666838486993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,32,0.003083733220895131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,128,0.0031669333577156065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1536,64,0.002935466667016347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,10240,0.020087466637293497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,8192,0.013697066903114319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,12288,0.02018666664759318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,16384,0.025332266092300416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,7168,0.014008532961209616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,6144,0.014165332913398743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,5120,0.013496533036231995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,4096,0.011248000462849935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,65536,0.06566933393478394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3584,0.01009386678536733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2048,0.006637866795063019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,3072,0.008843732873598735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,2560,0.007231999933719635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1536,0.005541333556175232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,1024,0.004769066472848257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,768,0.00420053352912267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,512,0.0037578667203585305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,256,0.003291733314593633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,128,0.0030005333324273427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,64,0.002948266764481862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,1024,32,0.002937600016593933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,16384,0.018252799908320107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,12288,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,10240,0.014498133460680643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,8192,0.012822399536768595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,7168,0.012030933300654094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,6144,0.012158933281898498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,5120,0.010684800148010255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,4096,0.009006933371225993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,65536,0.05405120054880778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3072,0.007125333448251088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2048,0.0058783998092015585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,3584,0.008423466483751934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,2560,0.00658133327960968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,768,0.004019200056791306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1024,0.004411733150482178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,1536,0.005145599941412607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,512,0.0036447999378045404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,256,0.0032501332461833954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,128,0.002979200085004171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,64,0.0029472000896930695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,768,32,0.002869333326816559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,16384,0.014084266622861228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,12288,0.012004266182581585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,10240,0.010507733623186747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,8192,0.009478400150934856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,7168,0.01209173301855723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,6144,0.009327999750773112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,4096,0.008413867155710856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,5120,0.009668266773223877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,65536,0.041883734862009685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3584,0.007996800045172375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,3072,0.007388799885908763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2048,0.005625600119431814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,2560,0.006386133531729381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1536,0.004903466502825419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,1024,0.004218666752179464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,768,0.00393599991997083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,512,0.0035797332723935447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,256,0.0031850665807724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,128,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,64,0.0028714666763941447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,512,32,0.002799999962250392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,8192,0.008444799979527792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,16384,0.01125440001487732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,10240,0.010223999619483948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,12288,0.010052266716957092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,7168,0.00767573316891988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,6144,0.008905599514643352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,5120,0.008249600231647492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,4096,0.007516799867153168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,65536,0.028985599676767986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3584,0.007222400108973186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1536,0.0048213332891464235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,3072,0.0068917334079742435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2048,0.005481599768002828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,2560,0.006222933530807495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,1024,0.004158933212359746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,512,0.0034986667335033415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,768,0.0037920000652472176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,256,0.0031669333577156065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,128,0.0029898665845394133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,64,0.002762666592995326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,256,32,0.003035733352104823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,10240,0.0075765331586201985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,12288,0.008236800134181977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,16384,0.0099263995885849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,8192,0.0071733335653940845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,65536,0.02360960046450297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,7168,0.006857599814732869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,6144,0.00666133314371109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,5120,0.0070592001080513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,4096,0.006748799979686737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3584,0.007109333574771881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2560,0.006049066781997681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,3072,0.006713599960009258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,2048,0.005369600156943003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1536,0.0047189335028330484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,1024,0.004068266600370407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,768,0.003735466549793879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,128,0.0029418667157491045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,256,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,512,0.00335359995563825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,64,0.002784000088771184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,128,32,0.002757333219051361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,12288,0.006818133095900218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,16384,0.007378133138020833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,10240,0.006926933427651723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,8192,0.006830933193365733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,6144,0.006534400085608165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,7168,0.006712533533573151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,65536,0.017786665757497152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,5120,0.0068896000583966565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,4096,0.006513066589832306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3584,0.006972800195217133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2048,0.005336533486843109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,3072,0.00661653329928716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,2560,0.006029866635799408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1536,0.004698666433493296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,1024,0.003996799886226654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,768,0.003651199986537298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,512,0.0032927999893824257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,256,0.002993066608905792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,128,0.0028714666763941447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,32,0.002762666592995326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,64,64,0.0027306665976842242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,12288,0.006586666901906331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,16384,0.006753066678841908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,8192,0.006775466601053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,10240,0.0068234667181968685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,65536,0.01225920021533966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,7168,0.0066101332505544026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,6144,0.006460799773534138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,5120,0.00680320014556249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,4096,0.006417066852251689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3584,0.006934399902820587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,3072,0.006523733337720235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2560,0.006030933558940887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,2048,0.005358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1536,0.004700799783070883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,1024,0.00402453343073527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,768,0.003655466685692469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,512,0.00335359995563825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,128,0.0029098667204380036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,256,0.002995199958483378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,64,0.002682666728893916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,256,32,32,0.0027200000981489818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,7168,0.3372447967529297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,8192,0.38135039011637367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,10240,0.4797973314921061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,12288,0.566651725769043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,5120,0.24650346438090004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,6144,0.29383786519368493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,4096,0.2127306620279948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,16384,0.744435183207194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3584,0.203058131535848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,3072,0.16327679951985677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2048,0.11157333056131999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,2560,0.14010666211446127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,256,0.0271071990331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1024,0.06497706572214762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,768,0.05818453232447306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,512,0.04075413147608439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,1536,0.09769813219706217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,128,0.01849386692047119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,64,0.014341333508491516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,65536,32,0.01447466711203257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,8192,0.09738667011260986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,10240,0.12056960264841717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,12288,0.14326292673746746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,16384,0.186026668548584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,7168,0.08597226937611899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,6144,0.07529173692067465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,5120,0.06397653420766194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,4096,0.059818665186564125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3584,0.04670826594034831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2560,0.035847465197245285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,3072,0.04458773136138916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,2048,0.029762132962544756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1536,0.02400533358256022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,768,0.013050666451454163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,1024,0.01688213348388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,256,0.007277866701285045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,512,0.010016000270843506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,128,0.0055061335364977515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,64,0.004833066463470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,32,0.005261866748332978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,10240,0.09197759628295898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,12288,0.10843520164489746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,16384,0.1400426705678304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,8192,0.07419626712799073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,7168,0.06556906700134277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,6144,0.06007039944330851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,4096,0.04017066558202108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,5120,0.0493343989054362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3584,0.036195198694864913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2560,0.02767360011736552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,2048,0.023053866624832154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,3072,0.03209493358929952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1024,0.012454400459925335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,768,0.010513066252072652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,1536,0.018388267358144125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,512,0.00842026670773824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,256,0.005656533439954122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,128,0.004648533463478088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,64,0.004349866509437561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,32,0.004599466423193614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,16384,65536,0.7144586563110351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,10240,0.09263786474863688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,12288,0.11322879791259766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,16384,0.12496533393859863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,8192,0.0653162678082784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,12288,65536,0.5719167709350585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,7168,0.0669653336207072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,6144,0.05105813344319662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,4096,0.0429802656173706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3584,0.032008532683054605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,5120,0.04627626736958822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,3072,0.02838933269182841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1024,0.01120853324731191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,1536,0.016221867005030314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2048,0.021186133225758873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,2560,0.02443839907646179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,128,0.004576000074545542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,768,0.00960533320903778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,512,0.007660800218582153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,256,0.0059125334024429325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,64,0.004117333392302195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,32,0.004583466549714406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,10240,0.06760426362355551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,12288,0.07795946598052979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,16384,0.09947946866353354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,10240,65536,0.47948586146036787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,8192,0.0518453319867452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,6144,0.04270506699879964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,7168,0.046607999006907146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,5120,0.040223999818166094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3072,0.02252266605695089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,4096,0.02799786726633708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,3584,0.02500373323758443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1536,0.012145066261291504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,1024,0.009347200393676758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2560,0.021447465817133585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,2048,0.015915733575820924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,768,0.008032000064849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,512,0.00652266691128413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,256,0.004459733267625173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,128,0.004001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,64,0.0037269333998362223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,32,0.00403413325548172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,10240,0.06683733463287353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,12288,0.06631360054016114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,16384,0.11044586499532064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,8192,65536,0.40368213653564455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,7168,0.04059413274129232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,8192,0.04493759870529175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,5120,0.0348853349685669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,6144,0.03579733371734619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3584,0.022900267442067464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,4096,0.02537173430124919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2560,0.017540266116460167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,3072,0.020463999112447104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,2048,0.014588800072669984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1536,0.011333333452542622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,1024,0.008995200196901958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,768,0.0076341331005096436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,512,0.006181333462397257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,128,0.004233600199222564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,64,0.0038783999780813852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,256,0.004709333181381226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,32,0.004194133480389913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,12288,0.057676800092061366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,10240,0.06751466592152913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,16384,0.0843786636988322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,7168,65536,0.3496074676513672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,8192,0.039112532138824464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,7168,0.03556266625722249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,5120,0.031091199318567915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,6144,0.03229119976361593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2560,0.015185067057609558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,4096,0.02235520084698995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3584,0.02025493383407593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,3072,0.018573866287867228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,2048,0.012648533781369528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1024,0.007906133433183034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,768,0.006866133213043213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,1536,0.010171733299891154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,512,0.005564799904823304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,256,0.004263466596603394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,128,0.0037439999481042228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,64,0.00348693331082662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,32,0.0037269333998362223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,10240,0.04315413236618042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,12288,0.051121068000793454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,16384,0.07343893051147461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,6144,65536,0.2983722686767578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,8192,0.03883093198140462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,5120,0.028097067276636762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,6144,0.030273065964380903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,7168,0.034439468383789064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3072,0.015709867080052696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2560,0.014593066771825156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,4096,0.021501866976420085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,3584,0.01990933418273926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,2048,0.012705066800117492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1024,0.007995733122030894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,1536,0.009173333644866943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,768,0.006931200126806895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,512,0.005264000097910563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,256,0.00411520004272461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,128,0.0037834666669368743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,64,0.0034453332424163817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,32,0.0034442665676275887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,10240,0.05268586476643881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,12288,0.04956586758295695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,5120,65536,0.24302825927734376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,16384,0.06439253489176432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,8192,0.03138453364372253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,5120,0.02183039983113607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,6144,0.025731199979782106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,7168,0.0326090673605601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3072,0.014254933595657349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,3584,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,4096,0.01681173245112101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2560,0.01165226697921753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1024,0.006554666658242543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,2048,0.01001706620057424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,1536,0.008221866687138875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,768,0.0057322666049003605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,512,0.00450133333603541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,128,0.0034847999612490333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,256,0.0039327998956044516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,64,0.003256533294916153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,32,0.0034527999659379324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,4096,65536,0.19932373364766437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,12288,0.04597760041554769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,10240,0.03227199912071228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,16384,0.049046401182810465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,8192,0.03598933219909668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,4096,0.02140480081240336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,6144,0.024821333090464272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,7168,0.02847893238067627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,5120,0.02076693375905355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3584,0.013848533233006796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,3072,0.015672533710797628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2560,0.012732799847920737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,2048,0.010985599954922994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1536,0.008358400066693623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,768,0.005493333439032236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,1024,0.006844800213972728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,512,0.0045503998796145115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,256,0.003853866706291834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,128,0.0035743998984495797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,64,0.0032810665667057036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,32,0.003499733408292135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3584,65536,0.1758794625600179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,12288,0.03337599833806355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,10240,0.032568534215291337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,16384,0.04589440027872722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,8192,0.02666986584663391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,7168,0.0240831991036733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,5120,0.020513067642847695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,4096,0.013429333766301474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,6144,0.021205333868662517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3584,0.0116074671347936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2560,0.01164906620979309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,3072,0.013429333766301474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,2048,0.009607467055320739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1536,0.008108800152937572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,768,0.005074133475621542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,512,0.004366933306058248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,1024,0.006243200103441874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,256,0.0037685332198937735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,65536,0.15336640675862628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,128,0.0034730667869249977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,64,0.0032672000428040824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,3072,32,0.0034783999125162757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,8192,0.020451200008392335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,10240,0.028678399324417115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,12288,0.029099732637405396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,16384,0.045015466213226316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,6144,0.01633280018965403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,5120,0.01695786714553833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,4096,0.011368532975514729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,7168,0.01857173244158427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3584,0.012049067020416259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,3072,0.011145599683125814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2560,0.00973653296629588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,2048,0.007414400080839793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,512,0.003885866701602936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1024,0.005471999943256378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,768,0.0043712000052134195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,1536,0.00639573335647583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,256,0.00359253336985906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,128,0.0032874666154384612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,32,0.003089066594839096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,65536,0.13503893216451007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2560,64,0.003159466634194056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,8192,0.019151999553044637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,10240,0.02892373402913411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,12288,0.03242986599604289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,16384,0.04010239839553833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,7168,0.017181867361068727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,6144,0.015092266599337259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,5120,0.017437867323557534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,4096,0.014379733800888061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3072,0.011238400141398113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,3584,0.012943999965985617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2560,0.009761066238085429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,2048,0.008500267068545024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1536,0.0058773333827654515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,1024,0.004660266637802124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,768,0.004273066421349844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,512,0.003852800031503042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,256,0.003419733295838038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,65536,0.1107103983561198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,128,0.0031466667850812277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,64,0.003045333425203959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,2048,32,0.0031648000081380212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,8192,0.01817173361778259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,16384,0.02940160036087036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,10240,0.020383999745051066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,12288,0.024486400683720908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,6144,0.014660267035166421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,5120,0.012802132964134216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,7168,0.016362667083740234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,4096,0.010916266838709514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3584,0.010053333640098572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2560,0.0077461332082748415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,3072,0.009180800120035807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,2048,0.00717439999183019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,65536,0.0899733304977417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1536,0.0060693333546320595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,768,0.0042346666256586705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,1024,0.004676266511281332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,512,0.0037962667644023894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,256,0.003319466610749563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,128,0.0031850665807724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,64,0.003011200080315272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1536,32,0.0029824001093705496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,16384,0.023861332734425863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,12288,0.01882773240407308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,10240,0.015723733107248943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,8192,0.013806933164596557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,7168,0.012876799702644348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,5120,0.010589866836865743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,6144,0.012009599804878235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,4096,0.009204266468683879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3584,0.00848746697107951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,65536,0.063100798924764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,3072,0.0075541332364082335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2560,0.0069482664267222095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,2048,0.0058890665570894875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1536,0.005156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,768,0.004003199934959412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,1024,0.004364799956480662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,512,0.0036042665441830954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,256,0.003260799994071325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,128,0.0029205332199732465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,64,0.0028618666032950084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,1024,32,0.0030048000315825146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,16384,0.017103999853134155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,10240,0.01534293293952942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,12288,0.01717546582221985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,8192,0.013491200407346091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,7168,0.013563733299573263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,4096,0.009290666381518046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,6144,0.012065066893895467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,5120,0.010666666428248088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,65536,0.050240000089009605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3584,0.008183466891447704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,3072,0.00735040009021759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2048,0.00582826683918635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,2560,0.006410666803518932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1536,0.005050666630268097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,1024,0.004374399781227112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,768,0.0040853333969910945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,512,0.0036277333895365395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,256,0.0032106667757034303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,128,0.0029877332349618276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,64,0.002899199972550074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,768,32,0.0028405333558718365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,16384,0.013991467157999673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,12288,0.013364266355832419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,8192,0.010912000139554342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,10240,0.012232533097267151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,7168,0.010378666718800863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,5120,0.00890880028406779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,6144,0.009620267152786254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,4096,0.007889066636562348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,65536,0.03896426757176717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3584,0.007574399809042613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,3072,0.00691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2560,0.00625600020090739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1536,0.004851200183232625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,2048,0.005541333556175232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,1024,0.00415786678592364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,768,0.00382080003619194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,512,0.003517866631348928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,256,0.0031285333136717476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,128,0.002924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,64,0.002850133428970973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,512,32,0.002773333340883255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,16384,0.013424000144004822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,12288,0.011105066537857056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,8192,0.008727467060089112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,10240,0.010305066903432209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,5120,0.007249066730340321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,65536,0.026841600735982258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,7168,0.008142933249473572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,6144,0.007707733412583668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,4096,0.006769066552321117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3584,0.007195733487606049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,3072,0.006861866513888042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2560,0.006211199859778086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,2048,0.005523199836413065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1536,0.004858666658401489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,1024,0.004177066683769226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,768,0.0038005332152048744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,512,0.0034495999415715536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,128,0.0029813334345817565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,256,0.0031114667654037476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,64,0.002825599908828735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,256,32,0.002842666705449422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,16384,0.009302399555842082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,12288,0.007997866471608479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,10240,0.007470933099587758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,8192,0.0070592001080513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,65536,0.019764266411463418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,7168,0.006849066913127899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,5120,0.007073066631952922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,6144,0.006621866424878438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,4096,0.006674133241176605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3584,0.0070271998643875126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,3072,0.006788266698519389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2560,0.006033066908518473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,2048,0.005362133185068766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1536,0.0046858668327331545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,1024,0.004037333279848098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,512,0.0033952000240484873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,768,0.0037290667494138084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,256,0.0030378667016824085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,128,0.002856533229351044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,64,0.0027669332921504974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,128,32,0.0028181334336598715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,16384,0.00763626645008723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,12288,0.0069472000002861025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,10240,0.006955733398596446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,8192,0.006841599941253662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,65536,0.015239466230074564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,6144,0.006485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,7168,0.006684799989064534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,5120,0.006866133213043213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,4096,0.006500266492366791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3584,0.006931200126806895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,3072,0.006604800124963124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2560,0.0060245335102081295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1536,0.00472320020198822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,2048,0.005358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,1024,0.004006399959325791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,768,0.0036661334335803984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,256,0.0031306666632493338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,512,0.0033503999312718712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,128,0.0028661333024501802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,64,0.0027285332481066385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,64,32,0.0026911998788515727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,16384,0.006793599824110668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,12288,0.0066453332702318835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,10240,0.0068693334857622785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,8192,0.006775466601053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,65536,0.01376106639703115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,6144,0.006419200201829274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,7168,0.006668800115585327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,4096,0.006420266628265381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,5120,0.0068010667959849044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3584,0.006842666864395141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,3072,0.006514133512973785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2048,0.005331199864546458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,2560,0.005976533393065134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1536,0.0046623999873797095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,768,0.0036245333651701607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,1024,0.004029866556326548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,512,0.0033514666060606635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,256,0.0030464000999927522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,128,0.0028607999285062153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,64,0.0026890667776266735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,192,32,32,0.0027413333455721537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,7168,0.3315711975097656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,8192,0.37875305811564125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,10240,0.47885440190633133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,12288,0.5649728139241537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,5120,0.24191786448160807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,6144,0.2898357391357422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,4096,0.21035307248433432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,16384,0.7399808247884114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3584,0.1984831968943278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2560,0.13191573619842528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,3072,0.16214292844136555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,2048,0.1051520029703776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1024,0.060271998246510826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,512,0.03800106843312581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,256,0.023299199342727662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,768,0.051354666550954185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,1536,0.09197759628295898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,64,0.010476799805959065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,128,0.013286399841308593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,65536,32,0.010148266951243084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,8192,0.09539519945780436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,10240,0.1178005297978719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,12288,0.13975359598795573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,16384,0.1814304033915202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,7168,0.08388586839040121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,5120,0.06282026767730713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,6144,0.0738047997156779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,4096,0.058667735258738196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2560,0.038118398189544676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3072,0.040378665924072264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,3584,0.045798401037851974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,768,0.01439573367436727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,2048,0.02902719974517822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1536,0.023432532946268715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,1024,0.016371200482050575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,512,0.00969493289788564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,256,0.007114666700363159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,128,0.005162666738033295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,64,0.004589866598447164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,32,0.0052373334765434265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,10240,0.09088426431020101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,12288,0.12451306978861491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,16384,0.15796480178833008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,7168,0.06524159908294677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,8192,0.07320533593495687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,6144,0.05987733205159505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,5120,0.04877760012944539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,4096,0.03938560088475545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3584,0.03504960139592488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,3072,0.031082665920257567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2560,0.026969599723815917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,2048,0.022437334060668945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1024,0.012647466858228049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,1536,0.017762132485707603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,768,0.010222933689753215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,512,0.008225066463152568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,256,0.005354666709899902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,128,0.004355200131734212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,64,0.003952000041802724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,32,0.004293333490689596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,16384,65536,0.7610634485880534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,10240,0.09161386489868165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,12288,0.09509332974751791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,16384,0.14335145950317382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,7168,0.05753173430760702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,8192,0.06361279884974161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,12288,65536,0.5632106781005859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,6144,0.05033493439356486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,5120,0.0428330659866333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3584,0.03489813407262166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,4096,0.038429868221282956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,3072,0.027474133173624675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1536,0.01527679959932963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,1024,0.011052800218264262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2560,0.02370133399963379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,2048,0.01960106690724691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,512,0.007845333218574524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,768,0.009505066275596618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,256,0.0052159999807675685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,128,0.004484266539414724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,64,0.003909333298603693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,32,0.00420053352912267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,10240,0.06161173184712728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,12288,0.07282346884409586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,16384,0.09313066800435385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,7168,0.04412906567255656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,8192,0.04933653275171916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,10240,65536,0.49272960027058915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,6144,0.039426132043202715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,4096,0.027448532978693645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3584,0.027852799495061236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,3072,0.02201706568400065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,5120,0.03357866605122884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1024,0.009276800354321798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,1536,0.012063999970753986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2560,0.018886399269104005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,2048,0.017435733477274576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,512,0.006497066716353099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,128,0.00386559988061587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,256,0.0043136000633239744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,768,0.007916800181070964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,64,0.003588266670703888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,32,0.003933866570393244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,10240,0.05467840035756429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,12288,0.06482986609141031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,16384,0.08282559712727865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,8192,65536,0.3906815846761068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,5120,0.034092799822489424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,8192,0.04396373430887858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,7168,0.03947946627934774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,6144,0.035249066352844236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3072,0.01976319948832194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2560,0.01700586676597595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,4096,0.024778666098912557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,3584,0.02239039937655131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,2048,0.013403733571370443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1024,0.008996267120043437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,1536,0.011142399907112122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,768,0.007521066566308339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,512,0.005866666634877523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,256,0.004276266694068909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,128,0.003818666686614355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,64,0.0037151999771595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,32,0.003798400113979975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,10240,0.04859946568806966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,12288,0.058024533589680995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,16384,0.07334720293680827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,7168,65536,0.34039360682169595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,7168,0.03543893496195476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,8192,0.03924800157546997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,6144,0.03148266673088074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,5120,0.03023359974225362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2560,0.015165866414705912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,4096,0.022013866901397706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3584,0.020113066832224528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,3072,0.01781760056813558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,2048,0.0125109334786733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1536,0.01034773290157318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,1024,0.008248533308506011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,768,0.0070165331164995836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,512,0.005160533388455709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,256,0.0042303999265035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,128,0.003804799914360046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,64,0.0034847999612490333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,32,0.0036917333801587426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,10240,0.04330879847208659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,6144,65536,0.29455572764078775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,12288,0.051123201847076416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,16384,0.07240533034006755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,8192,0.03501439889272054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,7168,0.035813331604003906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,6144,0.02802026669184367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,5120,0.024206932385762533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3072,0.015780267119407655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2560,0.013489066561063131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,4096,0.019879466295242308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,3584,0.018062933286031087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1536,0.009723732868830364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,2048,0.011194666226704914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,1024,0.007755733529726664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,768,0.006749866902828217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,512,0.005098666747411092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,64,0.0033312000334262846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,128,0.0037162666519482933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,256,0.004113066693147024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,32,0.003489066660404205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,12288,0.04653653303782145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,5120,65536,0.23865920702616372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,16384,0.056973866621653234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,10240,0.0402730663617452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,7168,0.035139199097951254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,5120,0.02050986687342326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,8192,0.034289065996805826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,6144,0.029262934128443403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3584,0.014006400108337402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,3072,0.012773332993189493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2560,0.0108842670917511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,4096,0.01563093364238739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,2048,0.009100799759229023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,768,0.005262933174769084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1536,0.007976533472537994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,1024,0.006246399879455566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,256,0.003909333298603693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,512,0.004309333364168803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,128,0.003442133218050003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,64,0.0032298666735490165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,32,0.0034527999659379324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,4096,65536,0.19440746307373047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,12288,0.041894400119781496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,10240,0.03681813478469849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,16384,0.051451734701792394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,8192,0.028842665751775104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,4096,0.013898666699727377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,6144,0.020296533902486168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,5120,0.017171200116475424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,7168,0.022669865687688192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3584,0.012618666887283326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,3072,0.011237333218256634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2560,0.009938133756319682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,2048,0.008591999610265095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1536,0.007445333401362102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,1024,0.005309866865475973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,768,0.004440533121426901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,256,0.0036127999424934386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,512,0.004004266609748204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,128,0.0033141332368055976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,64,0.0033088001112143196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,32,0.003352533280849457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3584,65536,0.1726784070332845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,10240,0.03275946577390035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,12288,0.037336532274882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,16384,0.04566613435745239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,8192,0.031652265787124635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,4096,0.015290666619936624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,5120,0.01984213391939799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,7168,0.0253493328889211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,6144,0.02255679965019226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3584,0.011602133512496948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,3072,0.01043839951356252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2560,0.009274666508038838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,2048,0.008025600016117096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1024,0.005147733290990194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,1536,0.007051733136177063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,768,0.004489600161711375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,512,0.003933866570393244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,128,0.0032597333192825317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,65536,0.15222080548604328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,64,0.003156266609827677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,256,0.003572266548871994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,3072,32,0.003260799994071325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,8192,0.024430932601292928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,10240,0.02879146734873454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,12288,0.03234240015347799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,16384,0.039850668112436934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,5120,0.016597333550453185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,6144,0.019962666432062785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,7168,0.0219050665696462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,4096,0.013313066959381104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3584,0.01220373312632243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,3072,0.010690133770306904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2560,0.008733866612116496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,2048,0.007758933305740357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1536,0.006477866570154827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,1024,0.0048426667849222815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,768,0.004473599791526795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,512,0.004040533304214477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,256,0.003446399917205175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,65536,0.1313173294067383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,128,0.0032405334214369455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,64,0.0030762667457262674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2560,32,0.003156266609827677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,10240,0.024477867285410564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,8192,0.0233952005704244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,12288,0.027893332640329997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,16384,0.03383359909057617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,7168,0.020537600914637247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,6144,0.018539732694625853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,4096,0.013776000340779623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,5120,0.016241066654523215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3584,0.011953066786130269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2560,0.00843946635723114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,3072,0.011052800218264262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,2048,0.0073077330986658735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,65536,0.1032480001449585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1024,0.004558933277924856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,1536,0.005884799857934316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,768,0.004149333387613296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,512,0.0037823999921480812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,256,0.0033439998825391137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,128,0.0031360000371932983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,64,0.002932266642649968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,2048,32,0.0031871999303499854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,10240,0.026583466927210492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,8192,0.02127679983774821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,12288,0.022629332542419434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,16384,0.027718400955200194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,7168,0.018959999084472656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,6144,0.017090133825937905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,5120,0.01514240006605784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,4096,0.012125866611798604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3584,0.01129919985930125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,3072,0.010191999872525533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,65536,0.08156267007191977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2560,0.007414400080839793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,2048,0.006348800162474315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1024,0.004442666471004486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,1536,0.005212800204753875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,768,0.004129066566626231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,512,0.0037258667250474296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,256,0.003305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,64,0.0029333333174387617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,128,0.003078400095303853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1536,32,0.0031445334355036415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,12288,0.020643200476964316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,10240,0.018118399381637573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,16384,0.021556266148885093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,8192,0.015081600348154704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,6144,0.012473600109418233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,7168,0.013614933689435324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,5120,0.011117866635322571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,4096,0.009689600268999735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3072,0.00809386670589447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,3584,0.008785067001978557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,65536,0.05949866771697998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2560,0.00728000005086263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,2048,0.006202666461467743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1536,0.005388799806435903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,1024,0.00446720023949941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,768,0.004051200052102407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,512,0.0036277333895365395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,256,0.003278933217128118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,128,0.003028266628583272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,64,0.0029546665648619336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,1024,32,0.0029493334392706556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,16384,0.022141865889231362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,12288,0.017794134219487508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,8192,0.013197867075602212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,10240,0.016478932897249856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,6144,0.01188373366991679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,7168,0.012840533256530761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,5120,0.010528000195821126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,65536,0.049193600813547775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,4096,0.009130666653315227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3584,0.008081066608428954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,3072,0.007053866485754649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2560,0.006365866462389629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,2048,0.005622399846712748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1536,0.004941866795221964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,1024,0.00421973317861557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,512,0.003576533248027166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,256,0.0031690667072931922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,768,0.003939199944337209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,128,0.0030591999491055804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,64,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,768,32,0.0029386666913827257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,16384,0.015460266669591268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,12288,0.014604799946149192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,10240,0.01309866706530253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,8192,0.011615999539693196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,7168,0.011079466342926026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,5120,0.009381332993507385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,65536,0.038076798121134445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,6144,0.0102101335922877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,4096,0.007990399996439617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3584,0.007933866480986278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2048,0.00553173323472341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,3072,0.007188266515731812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,2560,0.006213333209355672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1536,0.0048543999592463175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,1024,0.0041685332854588825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,768,0.003853866706291834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,128,0.003010133405526479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,512,0.0034688000877698263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,256,0.0030933332939942675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,64,0.002887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,512,32,0.0029279999434947968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,12288,0.009971200426419576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,16384,0.011381333072980244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,10240,0.009215999643007915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,8192,0.008281599978605907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,65536,0.02906879981358846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,6144,0.006954666475454967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,7168,0.007406933108965556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,5120,0.007242666681607564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,4096,0.00687360018491745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3584,0.007121066749095917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,3072,0.006818133095900218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2560,0.006086400151252747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,2048,0.005413333574930826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1536,0.004794666667779287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,1024,0.004107733319203059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,768,0.0037578667203585305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,512,0.0034005333979924522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,256,0.003092266619205475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,64,0.0028522667785485585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,128,0.0029365333418051405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,256,32,0.0028266665836175283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,16384,0.009065600236256917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,12288,0.007805866499741872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,65536,0.01691626707712809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,10240,0.007245866457621257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,8192,0.007096533477306366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,7168,0.0069461335738499955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,6144,0.006594133377075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,5120,0.00692799985408783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,4096,0.006678399940331777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3584,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,3072,0.006770133475462596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2560,0.006049066781997681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,2048,0.005354666709899902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1536,0.004711466530958811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,1024,0.004038399954636892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,512,0.0033759998778502146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,768,0.003701333453257879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,256,0.003019733230272929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,128,0.002918400118748347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,64,0.0027530667682488757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,128,32,0.0027818667391935987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,16384,0.007169066866238911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,10240,0.006958933174610138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,12288,0.006900266806284587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,8192,0.006822399795055389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,65536,0.014686933159828186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,7168,0.006614399949709575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,6144,0.006484266618887584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,5120,0.00694400022427241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,4096,0.006495999793211619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3584,0.00697813332080841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,3072,0.006636799871921539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2560,0.005985066791375478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,2048,0.005357866485913595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1536,0.004727466901143392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,1024,0.004011733333269755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,768,0.0036778666079044344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,512,0.003337600082159042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,128,0.0028661333024501802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,256,0.0029866665601730345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,32,0.0027424000203609467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,64,64,0.0027637332677841187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,12288,0.007227733234564463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,16384,0.006841599941253662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,10240,0.006851199766000111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,8192,0.006790400048096975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,65536,0.013194666306177775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,7168,0.006539733211199443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,5120,0.00684799998998642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,4096,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,6144,0.006471466521422069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3584,0.006828799843788147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2560,0.006004266440868378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,3072,0.006472533444563548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,2048,0.005335466563701629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1536,0.0046858668327331545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,1024,0.003984000037113826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,768,0.003605333218971888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,256,0.0029653333127498626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,512,0.0033557333052158357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,128,0.0029056000212828318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,64,0.0027104000250498454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,160,32,32,0.0026602665583292644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,7168,0.32723093032836914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,8192,0.37740373611450195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,10240,0.46647891998291013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,12288,0.5573663711547852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,6144,0.2843733469645182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,5120,0.23596372604370117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,4096,0.20542826652526855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3584,0.19472640355428059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,16384,0.7295136133829753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,3072,0.15517333348592122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2560,0.1428607940673828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,2048,0.10151253541310627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1024,0.06259200175603231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,512,0.034635734558105466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,256,0.020820266008377074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,1536,0.07990612983703613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,768,0.04925440152486165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,128,0.01171946624914805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,32,0.008714666962623597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,65536,64,0.00942186713218689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,8192,0.09442559878031412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,10240,0.11676586469014485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,12288,0.13944212595621747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,16384,0.18194026947021485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,7168,0.08321920235951742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,5120,0.06139093240102132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,6144,0.07266240119934082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,4096,0.057353599866231286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2560,0.03777920007705689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3584,0.04517226616541545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,3072,0.039212799072265624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,2048,0.027926399310429888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,512,0.009286399682362874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,768,0.01385599970817566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1024,0.015890133380889893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,1536,0.022356265783309938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,256,0.006696533163388569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,128,0.0046186665693918865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,64,0.004262400170167288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,32,0.004728533327579498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,10240,0.08805973529815674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,12288,0.10465706984202068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,16384,0.13581760724385578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,8192,0.07053013642628989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,7168,0.06270933151245117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,6144,0.05780693292617798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,4096,0.03859626849492391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,5120,0.046777598063151044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3584,0.034118398030598955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2560,0.026009599367777508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,3072,0.03038826584815979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,2048,0.021575466791788737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1024,0.012032000223795573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,768,0.00983893374602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,1536,0.017293866475423178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,512,0.0078005333741505934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,256,0.004974933465321859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,128,0.004124800115823746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,64,0.0038773333032925926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,32,0.004345599810282389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,16384,65536,0.7446079889933268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,10240,0.08667733669281005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,12288,0.08906026681264242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,16384,0.11488213539123535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,12288,65536,0.5524895985921223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,8192,0.06001173257827759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,7168,0.05318400065104166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,6144,0.04749439954757691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,5120,0.045848532517751055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3584,0.029207466046015422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,4096,0.032714666922887166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,3072,0.026130133867263795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1536,0.014428800344467163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2560,0.02517333428064982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,2048,0.018515199422836304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,1024,0.010450133681297302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,768,0.008833066622416178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,512,0.0069472000002861025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,256,0.0046965335806210835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,128,0.003974399964014689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,64,0.0037439999481042228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,32,0.004173866907755534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,10240,0.059879465897878015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,12288,0.07099733352661133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,16384,0.10475093523661297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,8192,0.04847040176391602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,7168,0.043036798636118576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,6144,0.0385973334312439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,10240,65536,0.47836694717407224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,4096,0.026733867327372235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,5120,0.032542934020360306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3584,0.023948800563812257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,3072,0.024152533213297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1024,0.010103467106819152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,1536,0.011876266201337178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2048,0.01514240006605784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,2560,0.018362667163213095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,768,0.007620266576608021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,512,0.00625600020090739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,256,0.004095999896526337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,128,0.003634133438269297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,64,0.0034293333689371743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,32,0.003764266769091288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,10240,0.05379306475321451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,12288,0.07274133364359538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,16384,0.09143253167470297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,8192,65536,0.3981962521870931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,8192,0.043188265959421795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,7168,0.038839467366536456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,6144,0.03470613161722819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,5120,0.03340906699498494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,4096,0.02390186587969462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3072,0.019003732999165853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,3584,0.021359999974568687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2560,0.016370133558909098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,2048,0.013345066706339517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1536,0.010544000069300334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,1024,0.00820906658967336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,768,0.007052800059318543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,512,0.004997333387533823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,64,0.003253333270549774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,256,0.003861333429813385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,128,0.0034485332667827605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,32,0.00360000009338061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,12288,0.054698665936787925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,10240,0.046266667048136395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,7168,65536,0.3447765350341797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,16384,0.07876479625701904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,8192,0.0374506672223409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,7168,0.03334506750106812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,6144,0.03406080007553101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,5120,0.025425066550572712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2560,0.01394773324330648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3584,0.018761599063873292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,3072,0.016637866695721946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,4096,0.02105493346850077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,2048,0.011595732967058818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1536,0.0095551997423172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,1024,0.007538133362929027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,768,0.006488533318042755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,512,0.004196266829967499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,256,0.0036992001036802924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,128,0.0032831999162832894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,64,0.003143466760714849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,32,0.0035562666753927866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,10240,0.04697386821111043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,12288,0.05409919818242391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,6144,65536,0.30841280619303385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,16384,0.06748480002085368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,5120,0.021939200162887574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,7168,0.02895680069923401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,8192,0.03237013419469197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,6144,0.02953386704126994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2560,0.012050132950146992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3072,0.013946666320164999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,4096,0.018196266889572144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,3584,0.016157866517702738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,2048,0.01018346647421519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1536,0.008577066659927367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,1024,0.006792533397674561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,768,0.005861333509286245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,512,0.004041599979003271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,256,0.0036415999134381616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,128,0.00323840007185936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,64,0.003151999910672506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,32,0.0034826666116714476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,5120,65536,0.25592746734619143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,16384,0.05749546686808268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,12288,0.03880746761957805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,10240,0.033768534660339355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,6144,0.021828265984853108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,7168,0.024488532543182374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,8192,0.03099306623140971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,5120,0.019139200448989868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3584,0.013471999764442444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,3072,0.012141866485277812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2560,0.010443733135859171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,4096,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,2048,0.009180800120035807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1536,0.007702399790287018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,1024,0.00602346658706665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,768,0.004771199822425842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,512,0.0042346666256586705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,256,0.003671466559171677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,64,0.003138133386770884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,128,0.0032885332902272543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,32,0.0033952000240484873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,4096,65536,0.19453439712524415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,12288,0.035224533081054686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,10240,0.029502934217453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,16384,0.05134613513946533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,8192,0.027669332424799603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,4096,0.013579733173052468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,6144,0.019368533293406168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,7168,0.021845332781473794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,5120,0.016684800386428833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3584,0.01223466694355011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2560,0.009947733084360758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,3072,0.011169067025184632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,2048,0.008598400155703227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1536,0.007456000149250031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,768,0.004731733103593191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,1024,0.005555200080076853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,512,0.0040832000474135075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,256,0.003549866626660029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,32,0.003352533280849457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,128,0.003305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,64,0.0031509332358837126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3584,65536,0.17137386004130045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,10240,0.030525867144266767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,12288,0.03499626715977987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,8192,0.02495573361714681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,16384,0.04292906522750854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,4096,0.012500266234079996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,5120,0.01653439998626709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,6144,0.017732266585032144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,7168,0.01984106699625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3584,0.011335466305414836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,3072,0.010762666662534077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2560,0.009241599837938945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,2048,0.008077866832415263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1536,0.007055999835332234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,1024,0.005070933202902476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,768,0.004572799801826477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,512,0.004048000027736028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,256,0.003587199995915095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,64,0.0030261332790056865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,128,0.003101866692304611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,32,0.0031957333286603295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,3072,65536,0.14683839480082195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,8192,0.02260800004005432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,10240,0.02405973275502523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,12288,0.028203733762105304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,16384,0.0387829343477885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,4096,0.011166933178901672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,5120,0.013339733084042868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,6144,0.015502933661142984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,7168,0.018081067005793254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3584,0.010297600428263347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,3072,0.009450667103131612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2560,0.008373333017031352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,2048,0.0074986666440963745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1024,0.004676266511281332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,1536,0.00622080018122991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,768,0.004259199897448222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,512,0.0038165333370367683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,65536,0.1252330700556437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,256,0.0034272000193595886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,128,0.003048533449570338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,64,0.002963199963172277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2560,32,0.0030805334448814393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,8192,0.018807466824849448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,12288,0.02577706575393677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,10240,0.022711465756098427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,16384,0.03235413432121277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,7168,0.016942934195200602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,6144,0.01444906691710154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,5120,0.01309866706530253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,4096,0.010854400197664897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3584,0.009718400239944459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,3072,0.009169066945711773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2560,0.007764266431331634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,2048,0.0068693334857622785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1536,0.005562666555245717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,1024,0.004533333579699198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,65536,0.10368320147196453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,768,0.004119466741879781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,512,0.0036298667391141256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,256,0.0033173332611719764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,128,0.003010133405526479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,64,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,2048,32,0.002946133414904277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,12288,0.020588799317677816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,16384,0.025246934096018477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,10240,0.017991467316945394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,8192,0.014986667037010192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,7168,0.013890133301417033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,6144,0.0123690664768219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,5120,0.010958932836850484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,4096,0.009502933422724406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3584,0.008827733000119527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,3072,0.008160000046094257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2560,0.007354666789372762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,2048,0.006321066617965698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,65536,0.08087893327077231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1536,0.005409066875775656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,512,0.0036789332826932275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,1024,0.00452159990866979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,768,0.004109866668780645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,256,0.003269333392381668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,128,0.0029887999097506206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,64,0.0029386666913827257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1536,32,0.0029877332349618276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,12288,0.014603733023007711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,16384,0.01869759956995646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,10240,0.013156267007191977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,8192,0.0114656001329422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,7168,0.012776533762613932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,6144,0.010269866387049357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,4096,0.009291733304659527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,5120,0.010156800349553425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3584,0.008669867118199667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,65536,0.056884264945983885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,3072,0.007997866471608479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2560,0.006450133522351582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,2048,0.005614933371543884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1536,0.004902400076389313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,1024,0.0042346666256586705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,512,0.003619199991226196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,768,0.0038805333276589714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,256,0.0032170665760835014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,128,0.002980266759792964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,64,0.002834133307139079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,1024,32,0.003010133405526479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,10240,0.012148267030715943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,16384,0.015099733074506124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,12288,0.013230933745702108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,8192,0.010896000266075134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,7168,0.010381866494814556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,5120,0.009406933188438415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,6144,0.009608532985051472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,4096,0.008165333171685536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,65536,0.0453877329826355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3584,0.00817493349313736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,3072,0.006962133447329204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2560,0.006242133180300395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,2048,0.0055167997876803074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1536,0.004826666911443075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,1024,0.004156800111134847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,768,0.0037952000896135964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,256,0.003156266609827677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,512,0.003455999990304311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,128,0.002961066613594691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,64,0.002780800064404805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,768,32,0.002845866729815801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,16384,0.012170666456222534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,12288,0.010726400216420491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,10240,0.01046720047791799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,8192,0.0093941330909729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,65536,0.034322134653727215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,6144,0.009751466910044353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,5120,0.008265600105126699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,7168,0.008657067020734151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,4096,0.007518933216730754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3072,0.006886399785677592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,3584,0.007218133409818013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2560,0.006236800054709116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,2048,0.005532800157864889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1536,0.004804266492525736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,1024,0.004137599964936575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,512,0.003470933437347412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,768,0.0037802666425704955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,256,0.0031221332649389905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,128,0.002906666696071625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,64,0.0029002666473388673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,512,32,0.0031178665657838186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,12288,0.00881813367207845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,16384,0.010514133175214132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,10240,0.007783466577529907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,8192,0.007412266731262207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,7168,0.0068800002336502075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,65536,0.02350506583849589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,6144,0.006668800115585327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,5120,0.007110400001207988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,4096,0.006729599833488464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3584,0.007086933155854543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,3072,0.00678719977537791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2560,0.006067200005054474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,2048,0.005403733253479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1536,0.004715733230113983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,1024,0.004068266600370407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,768,0.003703466554482778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,512,0.003401600072781245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,256,0.003123199939727783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,64,0.0027317332724730173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,128,0.002833066632350286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,256,32,0.0028149334092934927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,12288,0.0068245331446329755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,16384,0.007394133508205414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,10240,0.007003733515739441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,65536,0.015507200360298156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,8192,0.006906666855017345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,7168,0.006763733426729838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,6144,0.006570666531721751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,5120,0.006984533369541168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3584,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,4096,0.006592000027497609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,3072,0.006690133114655812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2560,0.006043733159701029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,2048,0.005362133185068766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1536,0.004710400104522705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,1024,0.0040501333773136135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,768,0.003639466563860575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,512,0.003382399926582972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,256,0.0030773334205150605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,128,0.002841600030660629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,32,0.0027477333943049112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,128,64,0.0027637332677841187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,16384,0.006770133475462596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,12288,0.006651733318964641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,10240,0.006821333368619282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,65536,0.011348266402880352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,8192,0.006806399921576183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,7168,0.006693333387374878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,6144,0.006485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,5120,0.006910933554172516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,4096,0.006419200201829274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3584,0.0069919998447100324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,3072,0.0066431999206542965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2048,0.005329066514968872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,2560,0.006028800209363302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1536,0.004664533336957296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,1024,0.004021333406368891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,768,0.003655466685692469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,512,0.0033141332368055976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,256,0.003028266628583272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,128,0.002881066749493281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,64,0.0027488000690937043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,64,32,0.0027082666754722597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,16384,0.00666133314371109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,12288,0.006552533308664958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,8192,0.006741333504517872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,10240,0.006791466474533081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,65536,0.010364799698193868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,6144,0.006367999811967213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,5120,0.00681279997030894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,7168,0.006572799881299336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3584,0.006704000135262807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,4096,0.006292266647020976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1536,0.004670933385690053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,3072,0.006343466540177663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2560,0.006007466713587443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,2048,0.005358933409055074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,1024,0.004046933352947235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,768,0.003676799933115641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,256,0.003044266750415166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,512,0.0033183999359607695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,128,0.002890666574239731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,64,0.0027456000447273255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,128,32,32,0.002631466587384542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,7168,0.32240638732910154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,8192,0.3662229220072428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,10240,0.45832319259643556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,12288,0.5587957382202149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,6144,0.2797408103942871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,5120,0.23318719863891602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,4096,0.20281707445780434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,16384,0.7223573048909505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3584,0.19218239784240723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2560,0.138919464747111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,3072,0.152019198735555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,2048,0.09885013103485107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,256,0.019795199235280357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,512,0.03431893189748128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,768,0.042081065972646076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,128,0.010478933652242024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1024,0.056194134553273524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,64,0.008188800017038981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,1536,0.08664106527964274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,65536,32,0.007399466633796692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,8192,0.09233919779459634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,10240,0.11397866408030193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,12288,0.1361610730489095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,16384,0.1796160062154134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,7168,0.08139946460723876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,6144,0.0713866631189982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,5120,0.06000746488571167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,4096,0.0567413330078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3584,0.04346453348795573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2560,0.032782934109369916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,3072,0.03830080032348633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,2048,0.027190399169921876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,768,0.011990400155385335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1024,0.015221333503723145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,1536,0.021613866090774536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,512,0.009064533313115438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,256,0.006422399977842967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,128,0.004237866898377737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,64,0.0039007999002933502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,32,0.0043029333154360454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,10240,0.08986453215281168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,12288,0.10229972998301189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,16384,0.14948906898498535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,7168,0.061595733960469565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,8192,0.0717141310373942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,6144,0.05727146863937378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,4096,0.03792320092519124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3584,0.03368533452351888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,5120,0.0460863987604777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2048,0.02132373253504435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,2560,0.025539199511210125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,3072,0.029947733879089354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,768,0.009895466764767965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1024,0.01199679970741272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,1536,0.01699733336766561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,256,0.0045514668027559916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,512,0.00782293329636256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,128,0.0038975998759269714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,32,0.003907199949026108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,16384,65536,0.7307701110839844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,64,0.003502933432658514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,10240,0.07372586727142334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,12288,0.10236586729685467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,16384,0.13211627006530763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,8192,0.059463465213775636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,7168,0.06113813320795695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,12288,65536,0.5413973490397136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,6144,0.04657813310623169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,4096,0.032568534215291337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,5120,0.03951573371887207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3584,0.03313279946645101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,3072,0.025560534000396727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1536,0.014427733421325684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2048,0.018639999628067016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,1024,0.010418132940928141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,2560,0.02206719915072123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,768,0.008896000186602275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,512,0.007213866710662842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,256,0.004299733539422353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,128,0.003828266759713491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,64,0.0035103999078273775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,32,0.0036127999424934386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,10240,0.05931200186411539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,12288,0.07036800384521484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,16384,0.10081173578898113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,8192,0.04812693198521932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,7168,0.04264426628748576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,6144,0.037817601362864176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,10240,65536,0.471834659576416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,5120,0.032364799578984575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,4096,0.02638079921404521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3584,0.025684267282485962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,3072,0.02371946573257446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1024,0.00986240009466807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,1536,0.011552000045776367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2560,0.01803626616795858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,2048,0.01460906664530436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,768,0.007473066449165344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,256,0.004177066683769226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,512,0.0058794667323430385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,128,0.0035904000202814737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,64,0.003425066669782003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,32,0.003689600030581156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,10240,0.0528661330540975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,12288,0.062344535191853845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,16384,0.08982933362325032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,8192,65536,0.38910080591837565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,8192,0.0428607980410258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,7168,0.03793066740036011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,6144,0.033779199918111166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,5120,0.032662399609883624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,4096,0.023727999130884806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3584,0.021229867140452066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,3072,0.018822399775187175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2560,0.016203733285268147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,2048,0.013338667154312134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1536,0.010641066233317058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,1024,0.008255999783674877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,768,0.0070261334379514056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,512,0.004886400202910105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,256,0.003881600002447764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,128,0.003420799970626831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,64,0.0034048000971476236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,32,0.0034517332911491393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,10240,0.04564053217569987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,12288,0.0544490655263265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,16384,0.08076480229695639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,7168,65536,0.33472318649291993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,8192,0.03714880148569743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,7168,0.03320213357607524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,5120,0.028585600852966308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,6144,0.029872000217437744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,4096,0.020837332804997763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2560,0.014021333058675131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3584,0.018732800086339315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,3072,0.016706132888793947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1536,0.00979306697845459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,2048,0.011677866180737812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,1024,0.00769706666469574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,768,0.006780800223350525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,512,0.004399999976158142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,256,0.0037248000502586366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,128,0.0033621333539485933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,64,0.0033717334270477297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,32,0.003446399917205175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,10240,0.039476267496744794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,6144,65536,0.28040959040323893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,12288,0.054103465874989834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,16384,0.0682591994603475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,5120,0.021783467133839926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,8192,0.03185919920603435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,7168,0.03293439944585164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,6144,0.0258026659488678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3072,0.014330666263898215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2560,0.012202666203180949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,3584,0.016210132837295534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,4096,0.01793173352877299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,2048,0.010386133193969726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1536,0.008726400136947633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,1024,0.006840533514817555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,768,0.005453866720199585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,512,0.004031999905904134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,128,0.0033013333876927694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,256,0.0036618667344252265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,64,0.003094399968783061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,32,0.00322026660044988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,5120,65536,0.23358826637268065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,10240,0.036974934736887614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,12288,0.043082666397094724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,16384,0.05319466590881348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,8192,0.03472746610641479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,6144,0.025870933135350542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,7168,0.02845653295516968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,5120,0.018329600493113198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,4096,0.015134933590888976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3072,0.01193173329035441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,3584,0.013406933347384135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2560,0.010397866368293762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,2048,0.009096533060073853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1536,0.007742933432261149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,1024,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,768,0.004332800209522247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,512,0.0037973334391911825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,256,0.00345920001467069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,128,0.0032490665713946023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,64,0.003078400095303853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,32,0.0033386667569478357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,4096,65536,0.18914772669474283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,10240,0.03323413332303365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,12288,0.038888533910115555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,16384,0.04796160062154134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,8192,0.031547733147939044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,4096,0.013206400473912559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,5120,0.01646613379319509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,6144,0.020489599307378134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,7168,0.02293333411216736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3584,0.011975466211636861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,3072,0.0109333336353302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2048,0.008504533767700195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,2560,0.0095551997423172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1536,0.007259733478228251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,1024,0.0051136001944541935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,768,0.004242133100827535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,512,0.0037674665451049804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,256,0.0034133332471052804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,128,0.0032287999987602235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,64,0.0029866665601730345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,65536,0.17488853136698407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3584,32,0.0031061333914597826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,8192,0.025329067309697466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,12288,0.0339626669883728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,10240,0.033088000615437825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,16384,0.04581439892450968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,5120,0.0146506667137146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,4096,0.011895466844240825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,6144,0.017283199230829875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,7168,0.021579732497533165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3584,0.011101866761843365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,3072,0.009937066833178203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2560,0.008955732981363932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,2048,0.007905066510041555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1536,0.006862933437029521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,1024,0.004737066725889841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,768,0.004204800228277842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,512,0.0037674665451049804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,256,0.003425066669782003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,128,0.0032106667757034303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,64,0.0029535998900731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,65536,0.14605226516723632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,3072,32,0.0031466667850812277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,10240,0.028578132390975952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,12288,0.029361067215601604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,16384,0.03638186852137248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,8192,0.021677867571512858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,5120,0.013022933403650919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,6144,0.0151936004559199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,4096,0.010942932963371278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,7168,0.017117865880330405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3584,0.009852799773216247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,3072,0.01042133371035258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2560,0.008460799853007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,2048,0.007336533566315968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1536,0.006242133180300395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,1024,0.004901333153247834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,768,0.004238933324813843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,512,0.0037578667203585305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,65536,0.12470719814300538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,256,0.003356799980004629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,128,0.0030005333324273427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,64,0.003011200080315272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2560,32,0.0030570665995279947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,12288,0.024972800413767496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,10240,0.021691733598709108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,8192,0.018149334192276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,16384,0.0306442658106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,7168,0.015874133507410685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,5120,0.012571733196576437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,4096,0.010364799698193868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,6144,0.014495999614397685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3584,0.009628799557685853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2560,0.007799466451009114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,3072,0.008967467149098714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,2048,0.007550933460394542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1536,0.005751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,65536,0.10226346651713054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,1024,0.0044725333650906885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,768,0.004081066697835922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,512,0.0036981334288915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,256,0.0032085334261258446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,64,0.00296426663796107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,128,0.0030432000756263735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,2048,32,0.003058133274316788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,8192,0.015043200055758158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,10240,0.01761173407236735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,12288,0.0202239990234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,16384,0.025226666529973345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,7168,0.013692800203959146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,6144,0.01234879990418752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,5120,0.010891733566919963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,4096,0.009473066528638203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3584,0.00886079967021942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,3072,0.008125866452852886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2048,0.006168533364931742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,2560,0.00728000005086263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,65536,0.08027306397755941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1536,0.005351466437180838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,1024,0.00444160004456838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,768,0.004147200038035711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,512,0.003676799933115641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,256,0.003319466610749563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,128,0.0030154667794704436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,64,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1536,32,0.0030122667551040648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,10240,0.014115200440088908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,12288,0.014973866939544677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,8192,0.012773332993189493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,16384,0.018514132499694823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,7168,0.011959466338157653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,6144,0.011261866490046183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,5120,0.010226133465766906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,4096,0.00941439966360728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,65536,0.054866135120391846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3584,0.008573866883913676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,3072,0.007701333363850911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2560,0.006244266529877981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,2048,0.005523199836413065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1536,0.004849066833655039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,1024,0.004172799984614054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,768,0.0038165333370367683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,512,0.0034645333886146545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,128,0.0030154667794704436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,256,0.003176533430814743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,32,0.0028437333802382152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,1024,64,0.0028533334533373516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,16384,0.01722773313522339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,10240,0.013048533598581949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,12288,0.014556800325711569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,8192,0.011749333143234253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,7168,0.010909866293271382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,65536,0.04460479815800984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,6144,0.01025279959042867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,5120,0.00937066674232483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,4096,0.00800960014263789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3584,0.007335466643174489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,3072,0.0071487997968991595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1536,0.0048426667849222815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2560,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,2048,0.005534933507442474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,1024,0.004112000018358231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,768,0.0038389332592487337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,512,0.0035103999078273775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,256,0.003160533308982849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,128,0.003033600002527237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,64,0.002820266783237457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,768,32,0.002792533238728841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,16384,0.013610666990280152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,12288,0.011783466736475626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,8192,0.009674666325251262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,10240,0.010703999797503154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,65536,0.03325226704279582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,7168,0.009154133001963298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,6144,0.00809279978275299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,5120,0.00735040009021759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,4096,0.006849066913127899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3584,0.007223466535409291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,3072,0.006858666737874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2048,0.005539200206597646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,2560,0.0062282666563987735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1536,0.004827733337879181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,1024,0.004156800111134847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,768,0.003804799914360046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,512,0.003442133218050003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,128,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,256,0.00315733328461647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,64,0.003033600002527237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,512,32,0.0027989332874615988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,12288,0.008379733562469483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,16384,0.010054399569829304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,10240,0.007762133578459422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,8192,0.0071263998746871945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,65536,0.022072533766428627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,5120,0.007113599777221679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,4096,0.006674133241176605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,7168,0.00691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,6144,0.00661653329928716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3584,0.007062399884064992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,3072,0.0067669332027435304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2560,0.006097066899140676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1536,0.004741333425045013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,2048,0.005402666827042898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,1024,0.004036266605059306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,768,0.003700266778469086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,512,0.003433600068092346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,128,0.0029898665845394133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,64,0.0027114666998386385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,256,0.0030975999931494398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,256,32,0.002776533365249634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,12288,0.006808533271153767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,16384,0.00747626672188441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,10240,0.006955733398596446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,8192,0.006898133456707001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,65536,0.013727999726931252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,7168,0.006687999765078227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,4096,0.006517333288987477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,6144,0.00659093310435613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,5120,0.006919466455777486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3584,0.007018666466077168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,3072,0.0066549330949783325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2560,0.006041599810123444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1024,0.004068266600370407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,2048,0.0054058666030565895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,1536,0.004712533454100291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,768,0.0036159999668598174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,256,0.003089066594839096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,512,0.0033919999996821085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,128,0.002937600016593933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,64,0.0027594665686289472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,128,32,0.0027274665733178455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,16384,0.006779733300209046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,8192,0.006749866902828217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,12288,0.006678399940331777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,10240,0.006836266815662384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,65536,0.011077333490053813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,7168,0.00664213349421819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,6144,0.006455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,4096,0.006427733103434245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3584,0.006910933554172516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,5120,0.006874666611353557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,3072,0.006546133259932201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2560,0.005998933315277099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,2048,0.005385600030422211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1024,0.004019200056791306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,1536,0.004678399860858917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,768,0.00365226666132609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,512,0.003304533412059148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,256,0.003036800026893616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,128,0.002926933268706004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,64,0.002754133443037669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,64,32,0.0027615999182065325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,65536,0.01049066682656606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,12288,0.00652266691128413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,10240,0.0068245331446329755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,16384,0.0066101332505544026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,8192,0.0068010667959849044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,5120,0.006797866523265838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3584,0.006681600213050842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,4096,0.006369066735108693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,7168,0.006477866570154827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,6144,0.006415999929110209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,3072,0.006388266881306966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2560,0.006025599936644236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,2048,0.005294933418432872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1024,0.003997866561015447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,1536,0.004682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,256,0.003070933371782303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,768,0.0036864000062147772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,512,0.003339733431736628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,64,0.00266239990790685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,128,0.002865066627661387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,96,32,32,0.0027200000981489818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,7168,0.33341118494669597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,8192,0.3804629325866699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,10240,0.4751402537027995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,12288,0.5733472188313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,5120,0.241373872756958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,6144,0.28821013768514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,4096,0.20789972941080728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3584,0.20224107106526695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,16384,0.7377034505208333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2048,0.09992319742838542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,3072,0.1564448038736979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,2560,0.1300319989522298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1024,0.06230719884236654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,768,0.044151465098063156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,512,0.03508373498916626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,256,0.02073813279469808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,1536,0.07914666334788004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,64,0.00860800047715505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,128,0.01106346646944682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,65536,32,0.007269333302974701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,8192,0.09233493010203044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,10240,0.11405759652455646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,12288,0.13674880663553873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,16384,0.1791594664255778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,7168,0.08146026929219564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,6144,0.07095466454823812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,5120,0.059596800804138185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,4096,0.056467199325561525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3584,0.04343680143356323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2560,0.03250666658083598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,3072,0.03792320092519124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,2048,0.026789333422978717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,768,0.011740799744923909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1024,0.015095466375350952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,1536,0.02134079933166504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,512,0.008968533078829447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,128,0.004057600100835165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,256,0.006244266529877981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,64,0.0038880000511805216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,32,0.0041354666153589886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,10240,0.08909760316212972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,12288,0.11082026958465577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,16384,0.14757013320922852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,7168,0.06629759867986043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,8192,0.07253333727518717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,6144,0.05804693301518758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,4096,0.038840532302856445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,5120,0.04837973515192668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3584,0.03538346687952677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2560,0.026652799050013228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,3072,0.03004053235054016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,2048,0.024784000714619954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1024,0.012295466661453248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,768,0.01034879982471466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,1536,0.017539199193318686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,512,0.007855999966462452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,256,0.004614399870236715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,128,0.0038250667353471124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,64,0.0034847999612490333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,32,0.003802666564782461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,16384,65536,0.7335061391194662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,12288,0.08799253304799398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,10240,0.08487679958343505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,16384,0.13080106576283773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,12288,65536,0.5410069147745769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,8192,0.0597760001818339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,7168,0.06037653287251791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,6144,0.04721173445383708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,5120,0.044854398568471274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3584,0.02844160000483195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,4096,0.032568534215291337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,3072,0.025550933678944905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2560,0.024654932816823325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1536,0.014242133498191834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,1024,0.010590933760007222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,2048,0.018154666821161906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,768,0.008854400118192036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,256,0.004226133227348328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,512,0.007119999825954437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,128,0.0037600000699361167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,64,0.003399466723203659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,32,0.0035904000202814737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,10240,0.05863893429438273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,12288,0.06980053583780924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,16384,0.10052693684895833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,8192,0.047566934426625566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,7168,0.04238933324813843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,10240,65536,0.4685301462809245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,6144,0.0380181352297465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3584,0.026532266537348432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,4096,0.026231465737024943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,5120,0.03173973361651103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,3072,0.0206986665725708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1536,0.011587199568748475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,1024,0.00876800020535787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2560,0.01993280053138733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,2048,0.014724266529083253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,512,0.005967999994754791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,768,0.007446399827798207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,256,0.004089600096146265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,128,0.0034815999368826545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,64,0.0033557333052158357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,32,0.0036586667100588477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,10240,0.05216319958368937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,12288,0.06203519900639852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,16384,0.08825173377990722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,8192,65536,0.3705120086669922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,8192,0.04237653414408366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,6144,0.03354453245798747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,7168,0.03766506512959798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,5120,0.03226773341496785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,4096,0.023550933599472045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3072,0.018681599696477254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,3584,0.021092265844345093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2560,0.01599360009034475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,2048,0.013148799538612366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1536,0.010573866963386535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,1024,0.008242133259773254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,768,0.006996266543865204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,512,0.0046079998215039575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,256,0.0037834666669368743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,64,0.003436800092458725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,128,0.003356799980004629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,32,0.0034773332377274835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,12288,0.053749334812164304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,10240,0.04516160090764364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,16384,0.07967466513315836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,7168,65536,0.33779306411743165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,8192,0.03672533432642619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,7168,0.032763733466466265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,5120,0.02834133307139079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,6144,0.02937600016593933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3584,0.01837546626726786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,4096,0.020667733748753865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2560,0.01397119959195455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,3072,0.016571733355522155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1536,0.009455999732017517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,1024,0.007763200004895528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,2048,0.011662933230400085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,768,0.006694399813810985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,512,0.004296533266703288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,256,0.003732266773780187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,128,0.003319466610749563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,64,0.003160533308982849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,32,0.0031786667803923286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,12288,0.05308906634648641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,10240,0.03878719806671142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,16384,0.0673408031463623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,6144,65536,0.285207462310791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,8192,0.03146239916483561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,7168,0.02834133307139079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,6144,0.025410133600234985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,5120,0.024195200204849242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3072,0.0139957328637441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2560,0.011992533008257549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,3584,0.015948800245920818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,4096,0.017835734287897746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,2048,0.010276266932487487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1536,0.008505599697430928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,768,0.005553066730499268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,1024,0.006866133213043213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,256,0.003601066768169403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,512,0.003995733211437861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,128,0.003269333392381668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,64,0.003052799900372823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,32,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,5120,65536,0.23353919982910157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,12288,0.04220373233159383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,10240,0.03643413384755452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,16384,0.05242026646931967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,8192,0.03415466547012329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,7168,0.02789120078086853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,5120,0.018308266003926595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,6144,0.02526293396949768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3584,0.013460266590118408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,3072,0.011911466717720032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2560,0.010550399621327717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,4096,0.01493013302485148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,2048,0.009084799885749817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1536,0.007626666625340779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,768,0.004298666616280874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,1024,0.006109866499900818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,512,0.0037813333173592886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,256,0.003437866767247518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,128,0.003156266609827677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,64,0.0030847998956839246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,32,0.00323840007185936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,4096,65536,0.18856746355692547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,12288,0.03821866512298584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,16384,0.049296001593271896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,10240,0.03264106710751851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,8192,0.03083946704864502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,4096,0.013596799969673157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,7168,0.02146880030632019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,6144,0.01909760038057963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,5120,0.016455466548601784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3584,0.012124799688657125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,3072,0.011028266946474711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2560,0.009645866354306538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,2048,0.008675199747085572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1024,0.005141333242257436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,768,0.004175999760627746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,1536,0.007377066711584728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,512,0.0037578667203585305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,256,0.003398400048414866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,128,0.0030965333183606463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,64,0.0029845332105954488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,32,0.002995199958483378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3584,65536,0.17296533584594725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,10240,0.02852906584739685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,12288,0.033054933945337935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,8192,0.02725546757380168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,16384,0.044257068634033205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,5120,0.01727679967880249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,4096,0.01511253317197164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,7168,0.022062933444976805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,6144,0.019725867112477622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3584,0.010966400305430096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,3072,0.009983999530474345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2560,0.008935466408729553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,2048,0.007896533111731212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1536,0.006811733543872833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,1024,0.004841599861780802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,768,0.004117333392302195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,512,0.003717333326737086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,256,0.0033749334514141084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,128,0.0031317333380381264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,64,0.0029279999434947968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,32,0.003017599880695343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,3072,65536,0.14491626421610515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,10240,0.025255467494328814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,12288,0.028907734155654907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,8192,0.02355413238207499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,16384,0.035854931672414145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,5120,0.013005866607030233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,6144,0.017222400506337485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,4096,0.010869333148002624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,7168,0.019326933224995933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3584,0.010134399930636088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,3072,0.009220266342163086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2560,0.008284799754619598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,2048,0.007299200197060903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1024,0.004465066889921824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,1536,0.006267733375231425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,768,0.004105599969625473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,65536,0.12358720302581787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,512,0.0037610667447249093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,256,0.003324799984693527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,64,0.0029258665939172106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,128,0.0030400000512599947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2560,32,0.0030069333811601003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,8192,0.01763199965159098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,10240,0.02127466599146525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,12288,0.024366933107376098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,16384,0.029908267656962077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,7168,0.016328533490498863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,6144,0.014868266383806863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,5120,0.014059733351071677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,4096,0.010852266351381938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3584,0.010728533069292705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2560,0.00769706666469574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,3072,0.008994133273760477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,2048,0.006821333368619282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,65536,0.1014954646428426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1536,0.005523199836413065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,1024,0.004278400043646494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,768,0.003950933367013932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,512,0.003587199995915095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,256,0.0032853332658608755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,64,0.00288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,128,0.0030229332546393077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,2048,32,0.0029493334392706556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,10240,0.017105066776275636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,8192,0.014198399583498635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,16384,0.024707200129826863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,12288,0.019462400674819948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,7168,0.013209600249926248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,6144,0.012139733632405598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,5120,0.011116799712181092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,4096,0.010078932841618855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3584,0.010763733585675558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,3072,0.00825386643409729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,65536,0.0793621301651001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2560,0.0072202667593956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,2048,0.006287999947865804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1536,0.00496319979429245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,1024,0.004261333247025808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,512,0.003636266787846883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,768,0.003956266740957896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,256,0.0032757334411144257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,128,0.0030410667260487873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,64,0.002899199972550074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1536,32,0.002902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,12288,0.014405333002408347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,10240,0.012502400080362954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,8192,0.011402666568756104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,16384,0.018074667453765868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,7168,0.010734933614730834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,6144,0.011833600203196208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,4096,0.009160533547401428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,5120,0.010541866223017376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,65536,0.05339093208312988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3584,0.0085098663965861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,3072,0.007773866752783458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2048,0.006019199887911478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,2560,0.006977066894372304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1536,0.00516480008761088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,1024,0.004241066674391428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,768,0.003902933249870936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,512,0.0035797332723935447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,256,0.0032127998769283296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,128,0.002962133288383484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,64,0.0028885332246621448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,1024,32,0.0028778667251269023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,16384,0.014834133783976236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,10240,0.013729066650072733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,12288,0.012328533331553142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,8192,0.011628799637158712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,6144,0.010496000448862713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,7168,0.01123306651910146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,5120,0.009491200248400371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,65536,0.04324906667073568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,4096,0.007999999821186066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3584,0.00783679982026418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,3072,0.007231999933719635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2048,0.00553706685702006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1536,0.0048650667071342465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,2560,0.006235733131567637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,768,0.0037994667887687682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,512,0.003470933437347412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,1024,0.004183466732501984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,256,0.0031498665610949195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,128,0.0028746667007605235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,64,0.002902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,768,32,0.002919466545184453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,16384,0.011742933591206869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,12288,0.010541866223017376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,10240,0.01092693308989207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,8192,0.00981333355108897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,6144,0.009757866462071735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,7168,0.00906986693541209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,65536,0.03208960096041362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,5120,0.00895146628220876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,4096,0.0076106667518615724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3584,0.007175466914971669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,3072,0.006851199766000111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2560,0.0061034664511680605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1536,0.004774400095144907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,2048,0.005423999826113383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,1024,0.004101333270470301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,768,0.003739733248949051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,512,0.0034741332133611047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,256,0.0030826665461063385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,128,0.0029535998900731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,64,0.0027583998938401537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,512,32,0.0027989332874615988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,16384,0.011075199643770853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,10240,0.008582400282224019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,12288,0.009717333316802978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,65536,0.020999467372894286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,8192,0.007348266740640004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,7168,0.007008000214894612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,6144,0.006668800115585327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,5120,0.007079466680685679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3072,0.006658133367697399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,4096,0.006716800232728322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,3584,0.006988800068696339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2048,0.005369600156943003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,2560,0.006050133208433787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1536,0.004731733103593191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,1024,0.004069333275159201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,512,0.0033781332274278007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,768,0.0036906667053699495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,256,0.002958933264017105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,128,0.0029504001140594482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,32,0.002775466690460841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,256,64,0.002780800064404805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,16384,0.007357866565386455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,12288,0.007401599983374278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,65536,0.013543466726938883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,10240,0.00697386662165324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,8192,0.006870399912198384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,6144,0.006500266492366791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,7168,0.0066655998428662615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,5120,0.006886399785677592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,4096,0.006489600241184235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3584,0.006972800195217133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,3072,0.006668800115585327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2560,0.006018133461475372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,2048,0.005356800059477488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1536,0.004699733356634776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,1024,0.004035199930270513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,512,0.0033002667129039764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,768,0.003667200108369192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,256,0.0030794667700926462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,128,0.0028319999575614927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,64,0.0028064000109831494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,128,32,0.0027072000006834666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,12288,0.006696533163388569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,16384,0.006702933212121327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,65536,0.012152533729871113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,8192,0.006693333387374878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,10240,0.006852266689141591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,7168,0.006600533425807953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,6144,0.006384000182151794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,5120,0.0068906664848327635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,4096,0.006421333551406861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3584,0.00691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,3072,0.006578133503595989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2048,0.005351466437180838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,2560,0.00598826656738917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1536,0.004667733112970988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,1024,0.004009599983692169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,768,0.0036831999818483984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,512,0.0033439998825391137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,256,0.003068800022204717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,128,0.0028245332340399425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,64,0.0027434666951497394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,64,32,0.0027530667682488757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,12288,0.006614399949709575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,65536,0.016357333461443583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,16384,0.0066538666685422255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,8192,0.00673173318306605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,10240,0.006762666503588359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,7168,0.006503466765085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,6144,0.0064085334539413456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,4096,0.006326400240262349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,5120,0.006760533154010773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3584,0.006850133339564006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,3072,0.006455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2560,0.006019199887911478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1536,0.0046517332394917805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,2048,0.005348266661167144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,1024,0.004018133382002512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,768,0.0036159999668598174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,512,0.0033386667569478357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,256,0.0030805334448814393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,128,0.002828799933195114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,64,0.0027274665733178455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,80,32,32,0.002699733277161916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,7168,0.33250773747762047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,8192,0.3795114517211914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,10240,0.4742698669433594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,12288,0.5696095784505208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,6144,0.2859999974568685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,5120,0.23983467419942223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,4096,0.2081226666768392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,16384,0.7395306905110677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3584,0.2010474681854248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2560,0.14531307220458983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,3072,0.1503050645192464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,2048,0.09790933132171631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,768,0.042345599333445234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,256,0.019526400168736777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1024,0.05570559899012247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,512,0.033479468027750654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,1536,0.08942506313323975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,128,0.010377599795659383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,64,0.007916800181070964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,65536,32,0.006444799900054932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,8192,0.10065600077311199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,10240,0.11526292959849041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,12288,0.14074026743570964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,16384,0.18352746963500977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,7168,0.08726720015207926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,6144,0.07122133572896322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,5120,0.06210666497548421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,4096,0.05560319821039835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2560,0.03180800080299377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3584,0.04304853280385335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,3072,0.04201066493988037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,2048,0.026261333624521894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,768,0.011516799529393513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1024,0.01467626690864563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,512,0.00879146655400594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,1536,0.02105493346850077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,256,0.006076799829800924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,128,0.004009599983692169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,64,0.0036917333801587426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,32,0.0038560000558694207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,10240,0.08536319732666016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,12288,0.10209706624348958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,16384,0.1346293290456136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,7168,0.0611135999361674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,8192,0.06907413005828858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,6144,0.05638613303502401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,4096,0.036992001533508304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,5120,0.04507840077082316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3584,0.03285013238588969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2560,0.02469546596209208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,3072,0.028860799471537274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,2048,0.0204693337281545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,768,0.009321600198745728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1024,0.011275733510653179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,1536,0.016251732905705772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,512,0.007301333546638489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,256,0.00410453329483668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,128,0.0036330667634805044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,16384,65536,0.7156981150309245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,64,0.0033674667278925574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,32,0.003551999976237615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,10240,0.07170453071594238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,12288,0.08562133312225342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,16384,0.131114665667216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,8192,0.05805333455403646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,7168,0.051541332403818765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,12288,65536,0.5574122746785481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,6144,0.05247253179550171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3072,0.024822400013605753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,3584,0.031990400950113934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,4096,0.03142506678899129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,5120,0.038225066661834714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1024,0.00997226635615031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,1536,0.013819733262062072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2048,0.0197653333346049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,2560,0.021205333868662517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,768,0.008291199803352356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,512,0.00661653329928716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,128,0.0035242666800816857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,256,0.0038912000755469004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,64,0.003373866776625315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,32,0.003469866762558619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,10240,0.05853653351465861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,12288,0.06935466925303141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,16384,0.09034240245819092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,8192,0.047432533899943036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,7168,0.04203626712163289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,10240,65536,0.46010773976643876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,6144,0.03738666772842407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,4096,0.026155734062194826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,5120,0.03143573403358459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3584,0.026331732670466106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,3072,0.020501333475112914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1024,0.00862613320350647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,1536,0.013103999694188437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2560,0.0176256000995636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,2048,0.014487466216087342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,768,0.007337599992752075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,512,0.005793066819508871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,256,0.0038176000118255613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,128,0.0034847999612490333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,64,0.003337600082159042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,32,0.0034186666210492453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,10240,0.0514357328414917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,12288,0.061459199587504065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,16384,0.08672853310902914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,8192,65536,0.367085870107015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,8192,0.04338879982630412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,7168,0.038389333089192706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,6144,0.0333621342976888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,5120,0.0321397324403127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2560,0.016288000345230102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,4096,0.023280000686645506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3072,0.018304000298182167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,3584,0.02147093415260315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,2048,0.01293333371480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1024,0.008344533046086629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,768,0.007050666709740956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,1536,0.010718933741251628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,512,0.004593066871166229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,128,0.003389866650104523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,256,0.0037077332536379496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,64,0.0032117334504922234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,32,0.0032000000278155005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,12288,0.05918613274892172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,10240,0.05013013283411662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,16384,0.07505599657694498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,7168,65536,0.33706026077270507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,8192,0.03715946674346924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,7168,0.0329749325911204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,5120,0.028034132719039918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,6144,0.029918932914733888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2560,0.01397119959195455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3584,0.018234666188557944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,4096,0.02044586737950643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,3072,0.016220800081888833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,2048,0.012098133563995361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1024,0.007585066556930542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,1536,0.009649067123730978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,768,0.006563200056552887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,512,0.004124800115823746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,256,0.003568000098069509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,128,0.0032490665713946023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,64,0.0031445334355036415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,32,0.00323840007185936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,12288,0.05003413359324137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,10240,0.03820693492889404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,16384,0.06291626691818238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,6144,65536,0.2754442532857259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,8192,0.03138773242632548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,6144,0.025149865945180254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,5120,0.02401813268661499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,7168,0.02804693380991618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3072,0.013883733749389648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2560,0.011874133348464965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,4096,0.017777067422866822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,3584,0.015845333536465965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,2048,0.010266666611035664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1536,0.008396800359090168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,1024,0.0068245331446329755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,768,0.005456000069777171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,512,0.0038592000802357995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,256,0.0033941333492596946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,128,0.0031776001056035364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,64,0.003017599880695343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,32,0.003050666550795237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,5120,65536,0.23100479443868002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,12288,0.04103786547978719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,10240,0.03522026538848877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,16384,0.05476266543070475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,8192,0.03317866722742717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,7168,0.026781866947809856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,5120,0.0180565337340037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,6144,0.021076265970865884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3584,0.013134933511416116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,4096,0.014874666929244995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,3072,0.011787733435630799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2560,0.010129066308339436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,2048,0.008982400099436443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1536,0.007669333120187123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,768,0.004265599946180979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,1024,0.006041599810123444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,512,0.0036938667297363283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,256,0.0033013333876927694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,128,0.003065599997838338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,64,0.0029535998900731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,32,0.0029290666182835894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,4096,65536,0.18772266705830892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,10240,0.03199466665585836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,12288,0.03779413302739461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,16384,0.04743359883626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,8192,0.030789333581924438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,6144,0.02216959993044535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,4096,0.013355732957522074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,7168,0.024807467063268026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,5120,0.01609493295351664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3584,0.011978666981061299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,3072,0.010918399691581726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2560,0.00955839951833089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,2048,0.008828799923261006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1536,0.007303466896216075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,1024,0.005097599824269613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,768,0.0040832000474135075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,512,0.003676799933115641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,256,0.003311999887228012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,64,0.002937600016593933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,128,0.003044266750415166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,32,0.00297173336148262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3584,65536,0.16214720408121747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,12288,0.033292800188064575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,8192,0.026397866010665894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,10240,0.028645332654317218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,16384,0.0407317320505778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,5120,0.014413866400718688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,4096,0.011895466844240825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,7168,0.021321600675582884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,6144,0.01691306630770365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3584,0.010967466235160827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,3072,0.009831466277440389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2048,0.007843199868996937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,2560,0.00885759989420573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1536,0.006838400165239971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,768,0.004060799876848856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,1024,0.004520533482233683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,512,0.0036650667587916053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,256,0.003305600086847941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,65536,0.143668270111084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,128,0.003050666550795237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,64,0.0028618666032950084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,3072,32,0.0029578665892283124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,10240,0.02459413409233093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,12288,0.028360533714294433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,8192,0.02058773239453634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,16384,0.03837440013885498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,6144,0.014841600259145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,4096,0.010790399710337321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,5120,0.012827733159065246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,7168,0.01685439944267273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3584,0.0098880002895991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,3072,0.00925546685854594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2560,0.008229333162307739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,2048,0.007167999943097432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1536,0.006250666578610738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,1024,0.004399999976158142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,768,0.004001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,512,0.0035914666950702667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,65536,0.12158186435699463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,256,0.003302400062481562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,128,0.003028266628583272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,64,0.0029120000700155893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2560,32,0.0028597332537174227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,12288,0.023625600337982177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,8192,0.017242666085561117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,16384,0.029388799269994097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,10240,0.02077440023422241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,7168,0.015719466408093772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,6144,0.014288000265757241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,4096,0.01112000048160553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,5120,0.012682666381200155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3584,0.008999466896057129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,3072,0.008504533767700195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2560,0.0075989335775375364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,2048,0.006728533407052357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1536,0.0051701332132021586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,1024,0.004192000130812327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,768,0.0038719999293486277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,65536,0.09866666793823242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,512,0.003553066651026408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,256,0.0032757334411144257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,128,0.0029781334102153777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,64,0.0028543998797734577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,2048,32,0.0028618666032950084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,8192,0.014178133010864258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,10240,0.016636799772580466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,12288,0.01913706660270691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,16384,0.024090667565663658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,7168,0.013195733229319254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,6144,0.012025599678357441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,5120,0.011212799946467083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,4096,0.010140800476074218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3584,0.00853760043780009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,65536,0.07585813204447428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2560,0.0070271998643875126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,3072,0.007805866499741872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,2048,0.005926399926344554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1536,0.004860800007979075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,1024,0.004178133110205332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,768,0.0038272000849246977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,512,0.003472000112136205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,256,0.003151999910672506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,128,0.0029824001093705496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,64,0.0028704000016053517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1536,32,0.0028234665592511495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,12288,0.0141184002161026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,16384,0.017863466342290243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,8192,0.011108266313870747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,10240,0.012821333607037863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,7168,0.010454400380452474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,6144,0.010587732990582783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,5120,0.009727999567985535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,4096,0.008946133653322856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,65536,0.05258880058924357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3584,0.007860266665617625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,3072,0.007331199944019318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2048,0.005532800157864889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,2560,0.006202666461467743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1536,0.004808533191680908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,768,0.0037994667887687682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,1024,0.004137599964936575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,512,0.003479466587305069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,256,0.0030773334205150605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,64,0.0029045333464940387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,128,0.0029578665892283124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,1024,32,0.0029088000456492106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,16384,0.014840533336003622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,12288,0.01237546702226003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,10240,0.01162986656030019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,8192,0.01025493343671163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,7168,0.009780266880989074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,6144,0.009142399827639262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,65536,0.04238506555557251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,5120,0.008197333415349324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,4096,0.0072202667593956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3584,0.007405866682529449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,3072,0.006875733534495036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2560,0.00614933321873347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,2048,0.00547733356555303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1536,0.0047647997736930845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,1024,0.004098133246103922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,768,0.003804799914360046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,512,0.003472000112136205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,128,0.0029567999144395193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,256,0.0030773334205150605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,64,0.0027882667879263563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,768,32,0.0028938665986061097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,16384,0.011703466375668842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,10240,0.009435733159383137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,12288,0.010342400272687275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,8192,0.008674133817354839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,7168,0.008006399869918824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,65536,0.03156053423881531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,6144,0.007112533350785573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,5120,0.007413333157698314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3584,0.007136000196139018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2560,0.006042666733264923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,3072,0.006832000116507213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,4096,0.00673280010620753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,2048,0.005389866729577383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1024,0.004062933226426443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,1536,0.004750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,768,0.0037162666519482933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,512,0.003416533271471659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,256,0.003050666550795237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,128,0.0029077333708604175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,32,0.0028149334092934927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,512,64,0.002820266783237457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,16384,0.008801066875457763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,12288,0.007282133400440216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,65536,0.019980800151824952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,10240,0.007143466671307881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,8192,0.006894933183987935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,6144,0.006563200056552887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,7168,0.00673280010620753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,5120,0.007009066641330719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,4096,0.006585599978764851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3584,0.007030400137106578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,3072,0.006669866542021434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2560,0.0060127998391787205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,2048,0.00537066658337911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1024,0.004046933352947235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,1536,0.004715733230113983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,768,0.0036949334045251214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,512,0.0033621333539485933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,32,0.00276799996693929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,64,0.0027306665976842242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,256,0.0030303999781608583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,256,128,0.002932266642649968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,65536,0.012312533458073933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,12288,0.006664533416430156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,16384,0.0067093332608540845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,8192,0.006772266825040181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,10240,0.006829866766929626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,7168,0.006600533425807953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,6144,0.0064735998709996535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3584,0.00687360018491745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,5120,0.006856533388296763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,4096,0.006419200201829274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,3072,0.006525866687297821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2560,0.006010666489601135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,2048,0.005353599786758423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1536,0.004694400231043497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,1024,0.004043733328580856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,768,0.003651199986537298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,512,0.003356799980004629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,128,0.002882133424282074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,256,0.003028266628583272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,64,0.0027530667682488757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,128,32,0.0027434666951497394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,65536,0.010763733585675558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,16384,0.006559999783833821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,10240,0.0067669332027435304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,12288,0.006569600105285645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,8192,0.006605866551399231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,7168,0.006512000163396199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,6144,0.006266666452089946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,5120,0.006727466483910878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3584,0.006712533533573151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,4096,0.006308266520500183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,3072,0.006487466891606649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2560,0.005965866645177205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,2048,0.005332266787687937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1536,0.004660266637802124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,1024,0.003978666663169861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,768,0.0036373332142829893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,512,0.0032437334458033243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,256,0.003065599997838338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,128,0.0028885332246621448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,64,0.002808533360560735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,64,32,0.0027445333699385325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,16384,0.006648533542950948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,65536,0.010797866185506185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,12288,0.006494933366775512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,8192,0.006593066453933716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,10240,0.006754133105278015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,6144,0.006227200229962667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,5120,0.006717866659164429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,7168,0.006412800153096516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,4096,0.0062613333264986675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3584,0.006584533552328746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1536,0.0046293333172798155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,3072,0.006239999830722809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2560,0.005973333120346069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,2048,0.005297066768010458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,1024,0.003969066590070724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,768,0.0036245333651701607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,512,0.003269333392381668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,256,0.0030261332790056865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,128,0.0028437333802382152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,64,0.0027402666707833606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,64,32,32,0.0027189334233601887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,7168,0.3183648109436035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,8192,0.3688096046447754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,10240,0.45615253448486326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,12288,0.5508629480997722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,5120,0.23009920120239258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,6144,0.2754645347595215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,4096,0.20191359519958496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,16384,0.7208703994750977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3584,0.1920970598856608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2560,0.12444266478220622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,3072,0.15001600583394367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,2048,0.09518933296203613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1536,0.08574506441752115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,1024,0.054231464862823486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,256,0.018787199258804323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,768,0.046277332305908206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,512,0.03246080080668132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,128,0.009828266501426697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,32,0.005776000022888183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,65536,64,0.0074538667996724445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,8192,0.09124266306559245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,10240,0.11174720128377277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,12288,0.13411733309427898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,16384,0.17626667022705078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,5120,0.0586357315381368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,7168,0.08032000064849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,6144,0.07013333638509114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,4096,0.05541546742121378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3072,0.03716799815495809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2560,0.03303253253300985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,3584,0.0424885352452596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,2048,0.026075732707977296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,768,0.013156267007191977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1024,0.01474453310171763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,512,0.008753066261609394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,1536,0.02059733271598816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,256,0.006018133461475372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,64,0.003555200000603994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,128,0.003822933385769526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,32,0.003664000084002813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,10240,0.09129599730173746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,12288,0.10920213063557942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,16384,0.1452895959218343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,7168,0.06537706851959228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,8192,0.07430613040924072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,6144,0.05651306708653768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,4096,0.03935786485671997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,5120,0.0474122683207194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3584,0.034663466612497966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2560,0.02584853370984395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,3072,0.03011626601219177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,2048,0.021476266781489055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,768,0.009891200065612792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1024,0.012040533622105916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,1536,0.016874667008717856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,512,0.007660800218582153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,256,0.004158933212359746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,128,0.003669333209594091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,16384,65536,0.7107029596964518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,64,0.00335359995563825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,32,0.0033887999753157297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,10240,0.07108906904856363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,12288,0.08654719988505045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,16384,0.13178773721059162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,12288,65536,0.5532202402750651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,7168,0.05204159816106161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,8192,0.057562665144602454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,6144,0.052407467365264894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,5120,0.04400320053100586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,4096,0.03174613316853841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3584,0.02785173257191976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,3072,0.024856533606847128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2560,0.02398080031077067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1536,0.01386240025361379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,1024,0.010175999999046326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,2048,0.01762666702270508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,768,0.008521599570910136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,512,0.006804266571998596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,256,0.0038399999340375268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,128,0.0035402665535608927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,64,0.0031946666538715364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,32,0.0032490665713946023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,10240,0.058302934964497885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,12288,0.06907306512196859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,16384,0.08988373279571533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,7168,0.04202880064646403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,8192,0.04724800189336141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,6144,0.037357866764068604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,10240,65536,0.45939200719197587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,5120,0.0313589334487915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3072,0.023035732905069987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,4096,0.025911466280619307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,3584,0.02301759918530782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1024,0.00862506628036499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,1536,0.0114847997824351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2560,0.017486933867136636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,2048,0.014366933703422546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,768,0.007315200070540111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,512,0.005726933479309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,256,0.003765333443880081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,128,0.0033728001018365227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,64,0.003336533407370249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,32,0.003323733309904734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,10240,0.051768533388773595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,12288,0.06161813338597616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,16384,0.0854634682337443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,8192,65536,0.36385920842488606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,8192,0.04206506808598836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,5120,0.03196159998575847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,6144,0.03336853186289469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,7168,0.03749866485595703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3072,0.018539732694625853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2560,0.016011733810106912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,3584,0.020890667041142782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,4096,0.023294933636983237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1024,0.008198399841785432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,1536,0.010707199573516846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,2048,0.013031466801961263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,768,0.0068917334079742435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,512,0.004403199752171834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,256,0.00360000009338061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,128,0.0032416000962257386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,64,0.0031061333914597826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,32,0.0031871999303499854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,10240,0.04558826684951782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,7168,65536,0.3401109377543131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,12288,0.056345601876576744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,16384,0.07915626366933187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,5120,0.025545599063237508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,7168,0.03808000087738037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,8192,0.03771626551946004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,6144,0.03017173409461975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2560,0.014653866489728292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3584,0.01885973413785299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,4096,0.021772799889246623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,3072,0.016634666919708253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,2048,0.012325333555539449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1536,0.009866666793823243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,1024,0.007702399790287018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,768,0.006378666559855144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,512,0.0040554667512575785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,256,0.003402666747570038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,128,0.00327360009153684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,64,0.003050666550795237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,32,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,10240,0.037859201431274414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,12288,0.045254401365915936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,6144,65536,0.2740778605143229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,16384,0.06382613182067871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,6144,0.028216532866160077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,8192,0.031032532453536987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,5120,0.021321600675582884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,7168,0.028122667471567792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3072,0.013928533593813578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2560,0.012056533495585125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,4096,0.017758933703104655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,3584,0.015589333573977151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,2048,0.010334933797518413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1536,0.008697600166002909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,1024,0.006871466835339864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,768,0.0054400001962979635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,512,0.0038272000849246977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,256,0.0034111998975276947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,128,0.0031850665807724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,64,0.0030645333230495454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,32,0.003045333425203959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,5120,65536,0.22694506645202636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,10240,0.03471680084864299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,16384,0.05053973197937012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,12288,0.040591998895009355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,8192,0.03256319959958394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,7168,0.0262442668279012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,6144,0.021027199427286782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,5120,0.017924267053604125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,4096,0.014745600024859109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3584,0.012989866733551025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,3072,0.01170133352279663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2560,0.010337066650390626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,2048,0.008946133653322856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1536,0.007533866663773854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,768,0.004138666639725367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,1024,0.006076799829800924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,512,0.003647999962170919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,256,0.003260799994071325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,128,0.0030229332546393077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,32,0.0030069333811601003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,64,0.002881066749493281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,4096,65536,0.18228160540262858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,12288,0.03640426794687907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,10240,0.031190399328867597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,8192,0.029357866446177168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,16384,0.04584213495254517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,4096,0.013237333297729493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,5120,0.016099199652671814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,7168,0.023849600553512575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,6144,0.018920532862345376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3584,0.012005333105723064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,3072,0.01088746686776479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2560,0.009677867094675701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,2048,0.008297599852085114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1536,0.007208533088366191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,1024,0.005100800096988678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,768,0.0040277334551016486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,512,0.003619199991226196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,256,0.0032992000381151833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,128,0.003054933249950409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,64,0.0028746667007605235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,32,0.002926933268706004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3584,65536,0.17008533477783203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,10240,0.028601600726445513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,12288,0.03252053260803223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,8192,0.026975999275843304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,16384,0.04069866736729939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,4096,0.011784533659617107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,5120,0.01735573410987854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,7168,0.02221226692199707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,6144,0.01939520041147868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3584,0.010872532924016316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,3072,0.00993173321088155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2560,0.008969600001970928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,2048,0.007810133198897044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1536,0.00680320014556249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,768,0.003994666785001755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,1024,0.004609066744645437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,512,0.00360959991812706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,256,0.0032768001159032187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,128,0.003110400090614955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,64,0.0029109333952267963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,65536,0.14098025957743326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,3072,32,0.00288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,8192,0.020387200514475505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,12288,0.027790933847427368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,10240,0.026848000288009644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,16384,0.034695466359456376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,6144,0.01651413341363271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,4096,0.012750933567682901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,5120,0.014371200402577718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,7168,0.01844693422317505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3584,0.011851732929547627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,3072,0.0090421328941981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2048,0.007166933516661327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,2560,0.008270933230717977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1536,0.005973333120346069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,1024,0.004402133325735727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,768,0.003931733220815659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,512,0.003585066646337509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,65536,0.11985493501027425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,256,0.003269333392381668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,128,0.003033600002527237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,64,0.0027903998891512555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2560,32,0.002946133414904277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,10240,0.020346667369206747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,12288,0.023423999547958374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,16384,0.028667734066645308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,8192,0.01688533425331116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,7168,0.015400532881418863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,5120,0.012642133235931396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,6144,0.014171733458836874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,4096,0.011106133460998535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3584,0.01064639985561371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,3072,0.008559999863306682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2048,0.006494933366775512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,2560,0.00759680022795995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1536,0.005184000233809153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,65536,0.09688639640808105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,1024,0.004193066557248434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,768,0.00383786658445994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,512,0.00352960005402565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,256,0.0031338666876157125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,128,0.0029418667157491045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,32,0.002811733384927114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,2048,64,0.002796799937884013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,10240,0.01651413341363271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,8192,0.014232533176740012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,16384,0.023191465934117635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,12288,0.01884373426437378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,5120,0.010859733819961548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,6144,0.01209493378798167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,4096,0.009213866790135701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,7168,0.01309866706530253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3584,0.008572799960772197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2560,0.007038933535416921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,3072,0.007835733393828075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,65536,0.0739573319753011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,2048,0.00582826683918635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1024,0.0041696002086003625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,1536,0.004836266736189524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,512,0.0035103999078273775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,768,0.003815466662247976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,256,0.003188266605138779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,32,0.0028927999238173166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,128,0.002946133414904277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1536,64,0.002733866622050603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,12288,0.014178133010864258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,16384,0.017313067118326822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,10240,0.012915199995040894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,8192,0.01135040024916331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,7168,0.010771200060844421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,4096,0.009022933244705201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,5120,0.009970133503278095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,6144,0.009710933764775593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,65536,0.051785600185394284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3584,0.008568533261617025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,3072,0.007227733234564463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2048,0.005545599758625031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,2560,0.0062496001521746315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1536,0.00484799991051356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,1024,0.004156800111134847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,768,0.0038090666135152185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,512,0.0035071998834609987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,256,0.0032085334261258446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,128,0.002898133297761281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,32,0.0028575999041398365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,1024,64,0.0029098667204380036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,16384,0.01418880025545756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,12288,0.012361600001653036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,10240,0.01109760006268819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,8192,0.011449600259462994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,7168,0.01071573297182719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,6144,0.010110933581988018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,5120,0.009169066945711773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,4096,0.007876266539096833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,65536,0.04133546749750773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3584,0.007222400108973186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2560,0.006121600170930227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,3072,0.006820266445477803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,2048,0.005423999826113383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1536,0.004770133395989736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,1024,0.0041002665956815084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,768,0.0037450666228930154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,512,0.0034261333445707956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,256,0.003202133377393087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,64,0.002796799937884013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,128,0.002935466667016347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,768,32,0.0027274665733178455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,8192,0.009521067142486572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,12288,0.01146346628665924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,16384,0.01151146690050761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,10240,0.010594133536020916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,65536,0.03076159954071045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,6144,0.007860266665617625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,7168,0.008756267031033833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,5120,0.007387733459472657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,4096,0.006651733318964641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3584,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,3072,0.006730666756629944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2560,0.006021333237489065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,2048,0.005393066505591074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1536,0.004700799783070883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,1024,0.0040394666294256846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,768,0.0037151999771595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,512,0.003324799984693527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,256,0.003147733211517334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,128,0.002885333448648453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,32,0.0027872001131375628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,512,64,0.002765866617361705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,16384,0.009807999928792317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,12288,0.008094933132330577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,10240,0.0077450667818387345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,65536,0.019117865959803262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,8192,0.006910933554172516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,7168,0.006785066425800323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3584,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,5120,0.0069930667678515124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,4096,0.006460799773534138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,6144,0.006520533561706543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,3072,0.006695466736952464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2560,0.006074666480223338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,2048,0.005364266534646353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1536,0.004710400104522705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,1024,0.0040277334551016486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,768,0.0036960000793139136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,512,0.003390933324893316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,256,0.0030048000315825146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,128,0.0027872001131375628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,32,0.0027562665442625684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,256,64,0.0027701333165168762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,12288,0.006680533289909363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,16384,0.006934399902820587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,10240,0.006819200019041698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,65536,0.013381333152453104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,8192,0.0067221333583196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,7168,0.00660159985224406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,6144,0.0065311998128890995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,4096,0.006514133512973785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,5120,0.006878933310508728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3584,0.006888533135255177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,3072,0.006563200056552887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2560,0.006020266811052958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,2048,0.005362133185068766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1536,0.004683733483155568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,1024,0.003991466760635376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,768,0.0036320000886917113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,512,0.0033717334270477297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,256,0.0030752000709374744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,128,0.0029056000212828318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,32,0.0026762666801611584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,128,64,0.0027104000250498454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,16384,0.006667733192443848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,65536,0.009985066453615824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,10240,0.006819200019041698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,12288,0.006533333162466686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,8192,0.006676266590754191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,7168,0.006548266609509785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,6144,0.0064181332786877945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,5120,0.006790400048096975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,4096,0.006402133405208588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3584,0.006858666737874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,3072,0.006534400085608165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2560,0.005982933441797892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,2048,0.005309866865475973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1536,0.004683733483155568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,768,0.003656533360481262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,1024,0.003961600114901861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,512,0.003306666761636734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,256,0.002924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,64,0.002755200117826462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,128,0.0028970666229724885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,64,32,0.002749866743882497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,16384,0.006595199803511302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,65536,0.008917333682378133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,12288,0.0065087998906771345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,10240,0.006631466746330261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,7168,0.006478933493296306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,8192,0.00661653329928716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,6144,0.006410666803518932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,5120,0.006782933572928111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2560,0.005962666869163513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3584,0.006717866659164429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,4096,0.006344533463319142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,3072,0.006391466657320659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1536,0.0046517332394917805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,2048,0.005276800195376078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,1024,0.003946666667858759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,768,0.003621333340803782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,512,0.0033269333342711128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,256,0.0030261332790056865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,64,0.002669866631428401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,128,0.002850133428970973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,48,32,32,0.0026837334036827086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,7168,0.3170314788818359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,8192,0.36441386540730797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,10240,0.4540416081746419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,12288,0.545315170288086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,5120,0.22786134084065757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,6144,0.2736384073893229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,4096,0.20058560371398926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,16384,0.7249322891235351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3584,0.19043413798014325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2048,0.09389440218607584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,2560,0.12316799958546956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,3072,0.14840639432271321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1024,0.052477868398030605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,768,0.03999040126800537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,512,0.03142506678899129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,256,0.01776960094769796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,128,0.009434666236241658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,1536,0.08474559783935547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,64,0.006949333349863689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,65536,32,0.005351466437180838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,8192,0.09047573407491048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,10240,0.11235626538594563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,12288,0.13540053367614746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,16384,0.17611947059631347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,6144,0.06973439852396647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,7168,0.07976000308990479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,5120,0.05864213307698568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,4096,0.055162668228149414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3584,0.04202666680018107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,3072,0.03674559990564982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2560,0.03133333325386047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,2048,0.02569813330968221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1024,0.014378666877746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,1536,0.023091200987497965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,768,0.013040000200271606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,512,0.00860800047715505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,256,0.005751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,128,0.0036703998843828833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,64,0.0033290666838486993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,32,0.0034485332667827605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,10240,0.08738666375478109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,12288,0.11012907028198242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,16384,0.1403018633524577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,8192,0.07141973177591959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,7168,0.06532479921976725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,6144,0.05482453505198161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,5120,0.045689598719278975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,4096,0.03898026545842488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3584,0.033316266536712644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2048,0.020544000466664634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,3072,0.02895039916038513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,2560,0.02555626630783081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1024,0.011658666531244914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,768,0.009684266646703084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,1536,0.016160000363985697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,512,0.007341866691907247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,256,0.003941333293914795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,128,0.003491200009981791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,64,0.003252266595760981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,32,0.0032821332414944967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,16384,65536,0.7040789286295573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,10240,0.08407039642333984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,12288,0.08415893713633218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,16384,0.11421653429667156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,12288,65536,0.5434122721354167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,7168,0.05067520141601563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,8192,0.0664682666460673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,6144,0.045823999245961505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3584,0.028075732787450153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,3072,0.02479893366495768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,5120,0.04380160172780355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,4096,0.031010133028030396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1536,0.015553067127863566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,1024,0.009942400455474853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2048,0.01744426687558492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,2560,0.021141332387924195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,768,0.008437333504358928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,512,0.00666133314371109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,256,0.0037119999527931214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,128,0.0033834666013717652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,64,0.0031199999153614043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,32,0.003094399968783061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,10240,0.058018132050832116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,12288,0.06860053539276123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,16384,0.0893514633178711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,10240,65536,0.45565013885498046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,7168,0.04157546758651733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,8192,0.047065599759419756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,6144,0.03673280080159505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,5120,0.035631998380025225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3584,0.022769065697987874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,3072,0.02012373407681783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,4096,0.025629866123199462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2560,0.019180800517400107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1536,0.011320533355077107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,1024,0.008515200018882752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,2048,0.014307199915250143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,768,0.007309866448243459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,512,0.005827199916044871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,256,0.0036277333895365395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,128,0.0033589333295822145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,64,0.0030794667700926462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,32,0.003236266722281774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,12288,0.061186134815216064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,10240,0.05151679913202921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,16384,0.07989973227183024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,8192,65536,0.3580597241719564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,7168,0.03718080123265584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,8192,0.04202026526133219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,5120,0.031802666187286374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,6144,0.032732800642649336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3584,0.02081386645634969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,4096,0.02309653361638387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,3072,0.01832746664683024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2560,0.01570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,2048,0.013029332955678305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1536,0.010597333312034607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,1024,0.008041599889596303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,768,0.0068234667181968685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,512,0.004316799839337667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,256,0.0034282666941483817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,128,0.0031776001056035364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,32,0.0029706666866938275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,64,0.0029738667110602063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,10240,0.04694826602935791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,12288,0.05421546697616577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,16384,0.07124160130818685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,7168,65536,0.31798505783081055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,7168,0.033317333459854125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,8192,0.03876693248748779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,6144,0.02949013312657674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,5120,0.028572799762090047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2560,0.014698666334152222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,4096,0.02097066640853882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3584,0.01925333340962728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,3072,0.016511999567349753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,2048,0.012274133165677388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1024,0.00751146674156189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,1536,0.00965226689974467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,768,0.0063274666666984555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,64,0.003031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,256,0.0033418667813142145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,128,0.0030634666482607523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,512,0.003885866701602936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,32,0.0029898665845394133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,12288,0.04509333372116089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,6144,65536,0.268395741780599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,10240,0.038387199242909745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,16384,0.05855466524759928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,5120,0.02114026745160421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,6144,0.02467413345972697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,8192,0.031203200419743855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,7168,0.031600000460942586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3072,0.01390720009803772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2560,0.011979732910792034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,3584,0.015773866573969522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,4096,0.01752426624298096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,2048,0.010281599561373393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1536,0.008593066533406576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,1024,0.006799999872843425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,768,0.005322666466236114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,256,0.003291733314593633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,512,0.003722666700681051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,128,0.00315733328461647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,64,0.0029333333174387617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,32,0.0029120000700155893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,5120,65536,0.22664106686909996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,12288,0.03968106508255005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,10240,0.03373440106709798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,16384,0.05005333423614502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,6144,0.02295253276824951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,7168,0.025354667504628496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,8192,0.031879466772079465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,5120,0.01765973369280497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3584,0.013078400492668152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,3072,0.011830400427182515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2560,0.010256000359853109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,4096,0.01458453337351481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,2048,0.008891733487447102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1536,0.007620266576608021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,1024,0.006001066664854685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,768,0.0040832000474135075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,512,0.0035232000052928926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,256,0.0032127998769283296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,128,0.0029685333371162414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,64,0.0028160000840822858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,32,0.002932266642649968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,4096,65536,0.18121280670166015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,12288,0.035778133074442546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,10240,0.03056959907213847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,16384,0.04470506509145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,8192,0.028586665789286297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,4096,0.013089066743850708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,7168,0.023407999674479166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,5120,0.018235733111699425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,6144,0.02096959948539734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3072,0.010707199573516846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,3584,0.011896533767382304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2560,0.009636267026265462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,2048,0.008471467097600301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1024,0.005055999755859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,1536,0.007143466671307881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,768,0.0039594667653242745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,512,0.0035562666753927866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,256,0.0032768001159032187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,128,0.003067733347415924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,64,0.002828799933195114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,32,0.0029205332199732465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3584,65536,0.16555627187093097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,10240,0.027530666192372637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,12288,0.03216746648152669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,8192,0.02623680035273234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,16384,0.041530664761861166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,7168,0.02118933399518331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,6144,0.018871466318766274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,5120,0.01669973333676656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,4096,0.01477226714293162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3072,0.009874133268992107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,3584,0.010853333274523418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2048,0.007769600053628286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,2560,0.008845866719881693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1536,0.006725333134333293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,768,0.0038954667747020722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,1024,0.004548266530036926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,512,0.003532800078392029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,256,0.0032127998769283296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,128,0.003005866706371307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,64,0.002799999962250392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,32,0.0028512001037597655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,3072,65536,0.14025600751241046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,12288,0.027342933416366576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,8192,0.022268799940745036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,10240,0.023717333873113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,16384,0.03408533334732056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,6144,0.01623146633307139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,4096,0.01249066690603892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,5120,0.014205867052078247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,7168,0.018093866109848023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3072,0.010918399691581726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,3584,0.01172266701857249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2560,0.008239999910195668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,2048,0.007088000078996022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1536,0.005970133344332377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,1024,0.004206933577855428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,768,0.0039007999002933502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,512,0.0034965333839257562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,65536,0.11909653345743816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,256,0.0031690667072931922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,128,0.0030421334008375804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,64,0.0028789333999156954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2560,32,0.002809600035349528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,16384,0.028092799584070842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,8192,0.016634666919708253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,12288,0.02278613249460856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,10240,0.02004479964574178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,6144,0.013779200116793313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,7168,0.015254400173823037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,5120,0.012434132893880208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,4096,0.010917333761850993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3584,0.010378666718800863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2560,0.007537066439787547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,3072,0.009744000434875489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,2048,0.0067552000284194945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,65536,0.09553813139597575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1536,0.005047466854254405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,1024,0.004134399940570196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,768,0.003765333443880081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,256,0.003172266731659571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,512,0.0034346667428811393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,128,0.0028768000503381092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,64,0.0028181334336598715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,2048,32,0.0027744000156720476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,10240,0.016499200463294984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,8192,0.013939199844996133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,12288,0.018449066082636516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,16384,0.0230293333530426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,7168,0.012984533111254373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,6144,0.01206826666990916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,5120,0.010822400450706482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,4096,0.009879466891288758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3584,0.009426132837931315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,3072,0.00879039963086446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2560,0.006761600077152252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,65536,0.07302400271097818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,2048,0.00581333339214325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1536,0.004794666667779287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,1024,0.0041514667371908825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,768,0.0037610667447249093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,512,0.003453866640726725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,128,0.002930133293072383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,256,0.00308693324526151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,64,0.0028362666567166646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1536,32,0.0028543998797734577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,10240,0.012785067160924276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,12288,0.013843199610710144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,16384,0.01720213294029236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,8192,0.011127466956774395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,7168,0.01051200032234192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,4096,0.008494933446248371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,5120,0.009250133236249288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,6144,0.009585066636403402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,65536,0.05027840137481689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3584,0.008138666550318401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,3072,0.0071712002158164975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2560,0.006088533500830332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,2048,0.005392000079154968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1536,0.004740266501903534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,1024,0.004065066576004028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,768,0.003673599908749262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,512,0.00342399999499321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,128,0.002882133424282074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,256,0.003089066594839096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,64,0.0028192001084486645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,1024,32,0.002754133443037669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,16384,0.014108799894650779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,12288,0.012089600165685017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,10240,0.01128000020980835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,8192,0.009995733698209126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,7168,0.009380267063776652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,6144,0.008646399776140849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,65536,0.04026559988657634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,4096,0.007134933272997539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,5120,0.008191999793052674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3584,0.00745066652695338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,3072,0.0067445332805315655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2560,0.0060693333546320595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,2048,0.005356800059477488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1024,0.0040448000033696495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,1536,0.004711466530958811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,768,0.0037237333754698435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,256,0.0029706666866938275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,512,0.0034154665966828666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,128,0.0028522667785485585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,64,0.002765866617361705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,768,32,0.0027829334139823914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,16384,0.011345066626866658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,12288,0.009893332918485005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,10240,0.009413333733876546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,7168,0.007693866888682048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,6144,0.007085866729418437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,65536,0.02980053424835205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,8192,0.00846613347530365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,5120,0.007101866602897644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,4096,0.006533333162466686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3584,0.007049599786599477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,3072,0.006673066814740498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2560,0.006027733286221823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,2048,0.005381333331267038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1536,0.004682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,1024,0.0040618665516376495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,768,0.003668266783157984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,512,0.0034186666210492453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,256,0.0031008000175158186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,128,0.0029418667157491045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,64,0.002733866622050603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,512,32,0.00275093341867129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,12288,0.007175466914971669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,10240,0.007063466807206471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,16384,0.008357333143552144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,65536,0.018288000424702962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,8192,0.006865066786607106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,6144,0.006520533561706543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,7168,0.006663466493288676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,5120,0.00689386675755183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,4096,0.006515199939409892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3584,0.006997333467006683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,3072,0.006704000135262807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2560,0.006021333237489065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,2048,0.005357866485913595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1536,0.004683733483155568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,1024,0.004021333406368891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,768,0.0036650667587916053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,128,0.0029311999678611755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,256,0.0029887999097506206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,512,0.0033770665526390077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,32,0.00271573339899381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,256,64,0.0027562665442625684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,16384,0.00668693333864212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,10240,0.006796800096829732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,65536,0.012269866466522217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,12288,0.006539733211199443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,8192,0.006584533552328746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,6144,0.006355200211207073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,7168,0.00647573322057724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3584,0.006738133231798808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,5120,0.006858666737874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,4096,0.006311466793219249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,3072,0.006493866443634033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2560,0.006031999985376993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,2048,0.005351466437180838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1536,0.004611200094223023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,1024,0.003988266736268997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,768,0.0036629334092140196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,512,0.0033418667813142145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,256,0.003058133274316788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,128,0.0028512001037597655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,64,0.0027488000690937043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,128,32,0.00275093341867129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,65536,0.012120532989501952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,16384,0.006537599861621857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,10240,0.0066453332702318835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,12288,0.006457599997520447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,8192,0.006555733581384023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,7168,0.006412800153096516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,5120,0.0065749332308769224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,6144,0.006171733140945435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,4096,0.006313600142796834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3584,0.00673280010620753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,3072,0.006405333181222279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2560,0.006005333364009857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,2048,0.0053162669142087305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1024,0.003986133386691412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,1536,0.004613333443800608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,768,0.003642666588226954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,512,0.0033258666594823206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,256,0.0030048000315825146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,128,0.0028490667541821797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,64,0.0026762666801611584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,64,32,0.0027306665976842242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,65536,0.010284800330797832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,16384,0.006589866677920024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,12288,0.006851199766000111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,8192,0.006469333171844482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,10240,0.006431999802589417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,7168,0.006333866715431213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,5120,0.006579199930032094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,6144,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,4096,0.0062282666563987735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3584,0.006578133503595989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,3072,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2560,0.0059562668204307554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,2048,0.00528959979613622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1536,0.00462719996770223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,1024,0.003934933245182038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,768,0.0036576000352700555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,512,0.0032885332902272543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,256,0.0028789333999156954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,128,0.0028405333558718365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,64,0.0026538667579491934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,32,32,32,0.002735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,7168,0.31541334788004555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,8192,0.36639359792073567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,10240,0.452515188852946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,12288,0.5433642705281575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,6144,0.27258132298787435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,5120,0.22840959231058755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,4096,0.19930240313212078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3584,0.18928640683492023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,16384,0.7141749064127605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2560,0.1361589272816976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,3072,0.1428127924601237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,2048,0.09254720211029052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1024,0.05646186669667562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,1536,0.07280960083007812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,64,0.006248533229033152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,128,0.009046399593353271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,256,0.01498240033785502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,512,0.03035306731859843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,768,0.03800853490829468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,65536,32,0.00468800018231074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,8192,0.08902826309204101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,10240,0.11117973327636718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,12288,0.1324661334355672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,16384,0.17438613573710124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,7168,0.07865920066833496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,6144,0.06908799807230631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,5120,0.057429333527882896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,4096,0.05379733244578043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3584,0.04144320090611776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2560,0.030845866600672407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,3072,0.03797760009765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,2048,0.025250132878621417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1536,0.01997440059979757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,1024,0.01416106621424357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,768,0.011290666460990906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,256,0.005721599857012431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,128,0.003475199888149897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,512,0.008426666259765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,64,0.0032277333239714304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,32,0.003252266595760981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,10240,0.08417920271555582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,12288,0.10760853290557862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,16384,0.13827412923177082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,8192,0.06961599985758463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,7168,0.06016853253046671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,6144,0.0542357325553894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,5120,0.044130134582519534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,4096,0.0361631989479065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3584,0.0327946662902832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,3072,0.02832213242848714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2048,0.02015786568323771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,2560,0.024475733439127602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1024,0.011520000298817952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,768,0.009618133306503296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,1536,0.01581653356552124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,512,0.007463466624418895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,256,0.003949866692225138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,128,0.0034058667719364167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,64,0.003197866678237915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,32,0.0031776001056035364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,16384,65536,0.7036021550496419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,10240,0.08307946523030599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,12288,0.08328853448232015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,16384,0.11297492980957032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,12288,65536,0.5368629455566406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,8192,0.058492799599965416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,7168,0.05019199848175049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,6144,0.04524906476338704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,4096,0.031113600730895995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,5120,0.04335999886194865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3072,0.024331732590993246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,3584,0.027102933327356978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1536,0.013516799608866373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,1024,0.009821866949399311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2048,0.019333332777023315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,2560,0.0207370658715566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,768,0.008341333270072937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,256,0.003587199995915095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,512,0.006513066589832306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,128,0.0032277333239714304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,64,0.0030570665995279947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,32,0.003031466652949651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,10240,0.056959998607635495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,12288,0.06814933617909749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,16384,0.08869333267211914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,10240,65536,0.4533568064371745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,8192,0.04632000128428142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,7168,0.041059199968973795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,6144,0.03654613494873047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3584,0.022373332579930624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,4096,0.025149865945180254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,3072,0.01981333295504252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,5120,0.03543039957682292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1536,0.011244799693425496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,1024,0.009591466188430786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2560,0.017037866512934367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,2048,0.014018133282661438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,768,0.007230933507283528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,512,0.005623466769854228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,256,0.0035904000202814737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,128,0.0032320000231266023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,64,0.003068800022204717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,32,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,10240,0.054366934299469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,12288,0.06414613326390585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,16384,0.08212160269419352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,8192,65536,0.36282666524251306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,8192,0.04142400026321411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,7168,0.03678826491038005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,6144,0.03246080080668132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,5120,0.03141653339068095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,4096,0.022734934091567995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3584,0.020397865772247316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,3072,0.01816533406575521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2560,0.015493333339691162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,2048,0.012753066420555115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1536,0.010541866223017376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,1024,0.007971199850241344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,128,0.0031648000081380212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,256,0.0033919999996821085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,512,0.004147200038035711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,768,0.006746666630109151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,64,0.0029813334345817565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,32,0.002917333443959554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,10240,0.04474773406982422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,12288,0.05516159931818644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,16384,0.07304213047027588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,7168,65536,0.3069354693094889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,8192,0.03679999907811483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,7168,0.03345066706339518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,5120,0.028326400121053058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,6144,0.029341866572697956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2560,0.014294399817784628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3072,0.016539733608563742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,4096,0.020536533991495767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,3584,0.018879999717076622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1024,0.0072970668474833175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,1536,0.009834667046864826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,2048,0.012169599533081055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,768,0.006361599763234456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,512,0.0038890667259693147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,256,0.0033088001112143196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,64,0.0029120000700155893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,128,0.0030389333764712016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,32,0.002948266764481862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,6144,65536,0.26860694885253905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,12288,0.044171734650929766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,10240,0.037929598490397134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,16384,0.05830080111821493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,8192,0.03561066786448161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,5120,0.020835200945536293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,6144,0.024397865931193034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,7168,0.02714879910151164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,4096,0.017318399747212727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3072,0.013682132959365845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2560,0.011735467116038005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,3584,0.01542080044746399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,2048,0.010214400291442872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1536,0.008572799960772197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,1024,0.006667733192443848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,768,0.0052821333209673565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,512,0.0035936000446478524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,256,0.0032042667269706728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,128,0.0030741333961486817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,64,0.0029056000212828318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,32,0.0028522667785485585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,5120,65536,0.2235520044962565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,12288,0.03914026816685994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,10240,0.03333119948705037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,16384,0.04934933185577393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,8192,0.031190399328867597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,6144,0.022410666942596434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,7168,0.02500266631444295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,5120,0.01749546726544698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3584,0.014929067095120749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,3072,0.01162986656030019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2560,0.01035520037015279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,4096,0.01639253298441569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,2048,0.008949333429336548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1536,0.007502933343251546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,1024,0.005915733178456625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,768,0.0038176000118255613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,256,0.0032149332265059153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,512,0.003537066777547201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,128,0.0030154667794704436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,64,0.0029077333708604175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,32,0.003013333429892858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,4096,65536,0.17762239774068195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,8192,0.028226133187611895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,10240,0.03004693388938904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,16384,0.04405333201090495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,12288,0.03460799853006999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,4096,0.015019733707110086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,6144,0.020504534244537354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,7168,0.022922666867574056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,5120,0.017723733186721803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3584,0.01360319952170054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2560,0.009603200356165568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,3072,0.010664533575375874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,2048,0.008414933085441589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1536,0.0071712002158164975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,1024,0.004755199948946635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,768,0.0038912000755469004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,512,0.0035232000052928926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,256,0.0032320000231266023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,128,0.002961066613594691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,65536,0.162446928024292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,64,0.0028629332780838014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3584,32,0.0029045333464940387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,10240,0.027478400866190595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,8192,0.023512534300486245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,16384,0.03829653263092041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,12288,0.03442453145980835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,5120,0.015998933712641397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,4096,0.014050133029619851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,7168,0.021223467588424683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,6144,0.01889386574427287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3584,0.012940800189971924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,3072,0.011762133240699768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2560,0.00897920032342275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,2048,0.007663999994595845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1536,0.006663466493288676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,1024,0.004392533500989278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,768,0.003857066730658213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,512,0.003489066660404205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,256,0.0032426667710145317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,128,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,65536,0.13864213625590008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,64,0.0028405333558718365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,3072,32,0.0028575999041398365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,10240,0.023312000433603923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,12288,0.030036266644795733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,16384,0.03352533181508382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,8192,0.019374932845433554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,6144,0.01591679950555166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,5120,0.014101333419481912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,4096,0.012146133184432983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,7168,0.01789439916610718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3584,0.011522133151690166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,3072,0.010722133517265319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2048,0.007086933155854543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,2560,0.00820479989051819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1024,0.004126933217048645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,768,0.0038122666378815973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,1536,0.005884799857934316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,65536,0.11647679805755615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,512,0.00347626656293869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,256,0.0031178665657838186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,128,0.003017599880695343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,32,0.002869333326816559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2560,64,0.002865066627661387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,8192,0.01627413332462311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,10240,0.019502933820088705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,12288,0.02224853237469991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,16384,0.02762346665064494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,7168,0.014894933501879374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,4096,0.010834133625030518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,6144,0.013435733318328858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,5120,0.012346667051315308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3584,0.010263466835021972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,3072,0.009617066383361817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2048,0.006502399841944377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,2560,0.007493333518505096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,65536,0.09392746289571127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1024,0.004105599969625473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,1536,0.004756266872088114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,512,0.0034154665966828666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,768,0.0037290667494138084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,128,0.003101866692304611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,256,0.00317546675602595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,64,0.0027583998938401537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,2048,32,0.002780800064404805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,10240,0.01618880033493042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,12288,0.018437333901723228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,16384,0.022457599639892578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,8192,0.013590400417645773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,7168,0.012857600053151449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,6144,0.011912533640861511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,5120,0.010594133536020916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,4096,0.00965013305346171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3584,0.009266133109728496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2560,0.0068906664848327635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,3072,0.008775466680526733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,2048,0.005582933127880096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,65536,0.07243413130442301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1536,0.004745600124200186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,1024,0.0040448000033696495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,768,0.0037290667494138084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,512,0.0034111998975276947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,256,0.0030965333183606463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,128,0.002946133414904277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,64,0.0027317332724730173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1536,32,0.002799999962250392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,16384,0.016977065801620485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,10240,0.012601600090662638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,8192,0.011011200149854024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,12288,0.013911466797192892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,6144,0.009404800335566203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,7168,0.010518399874369304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,5120,0.009001599748929341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,65536,0.049218134085337324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,4096,0.00830506682395935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3584,0.007703466713428498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,3072,0.0069909334182739254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2560,0.006052266558011373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,2048,0.005420800050099691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1536,0.0047423998514811196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,1024,0.004065066576004028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,768,0.003706666578849157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,512,0.0033749334514141084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,256,0.0030389333764712016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,128,0.0028927999238173166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,64,0.0028031999866167706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,1024,32,0.0028351999819278715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,16384,0.013851733009020487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,12288,0.012129066387812297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,10240,0.010890666643778484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,8192,0.009973333279291789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,7168,0.009340799848238627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,6144,0.00853760043780009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,65536,0.03919680118560791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,5120,0.007986133297284443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,4096,0.006904533505439759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3584,0.007309866448243459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,3072,0.006648533542950948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2560,0.006043733159701029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1536,0.004761599997679392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,2048,0.0053493330876032506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,1024,0.004070399949947992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,512,0.0034005333979924522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,768,0.003738666574160258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,256,0.003123199939727783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,64,0.0027583998938401537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,128,0.0028181334336598715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,768,32,0.0027637332677841187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,12288,0.00981333355108897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,16384,0.011230933666229247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,10240,0.009300266702969868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,8192,0.008406399687131246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,65536,0.028836266199747725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,7168,0.007303466896216075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,6144,0.006833066542943318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,5120,0.00698773314555486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,4096,0.006445866823196411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3072,0.006609066824118296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,3584,0.007008000214894612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2560,0.0060479998588562015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,2048,0.005345066885153452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,512,0.003389866650104523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1536,0.004678399860858917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,1024,0.004025600105524063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,768,0.003702399879693985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,256,0.00296426663796107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,128,0.0029696000119050344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,32,0.0027701333165168762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,512,64,0.0027647999425729113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,16384,0.008338133494059246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,65536,0.017771732807159425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,10240,0.00699946681658427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,12288,0.006995200117429097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,7168,0.006505600114663441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,8192,0.0066442668437957765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,6144,0.006439466774463653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,5120,0.006850133339564006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3584,0.006781866649786632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,4096,0.00639466643333435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,3072,0.006568533182144165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2560,0.006039466460545858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,2048,0.005342933535575867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1536,0.004711466530958811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,1024,0.004030933231115341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,768,0.003647999962170919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,256,0.0030879999200503034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,512,0.0033717334270477297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,128,0.002846933404604594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,64,0.0027295999228954316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,256,32,0.002700799951950709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,12288,0.006485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,65536,0.012534399827321371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,10240,0.006708266834417979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,16384,0.006623999774456024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,8192,0.006622933348019918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,7168,0.006313600142796834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,6144,0.0062826668222745255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,5120,0.0065653334061304735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,4096,0.006262399752934774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3072,0.0063391998410224915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2560,0.005773866673310598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,3584,0.006750933329264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,2048,0.005161599814891815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1536,0.004545066754023234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,1024,0.003909333298603693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,768,0.0036373332142829893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,256,0.002974933385848999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,512,0.003310933212439219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,128,0.0028864001234372456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,64,0.0026943999032179515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,128,32,0.002701866626739502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,65536,0.009765332937240601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,16384,0.006384000182151794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,10240,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,12288,0.006413866579532623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,8192,0.006444799900054932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,7168,0.006226133306821187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,6144,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,5120,0.006505600114663441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,4096,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3584,0.006585599978764851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,3072,0.0062613333264986675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2560,0.0056309332450230915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1536,0.004461866617202759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,2048,0.005026133358478546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,1024,0.003836799909671148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,768,0.0035413332283496858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,64,0.0026687999566396077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,512,0.0032159999012947083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,128,0.002780800064404805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,256,0.002948266764481862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,64,32,0.002700799951950709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,12288,0.00624533345301946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,65536,0.009525332848230999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,8192,0.006346666812896728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,16384,0.006362666686375936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,10240,0.00643093337615331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,5120,0.006428800026575724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,7168,0.0062047998110453285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,6144,0.00609493354956309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,4096,0.006054399907588959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3584,0.006549333532651265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,3072,0.0061930666367212926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2560,0.005636266867319743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,2048,0.005065600077311197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1024,0.0038304001092910765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,1536,0.0043935999274253845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,768,0.003568000098069509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,256,0.002961066613594691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,512,0.0031466667850812277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,128,0.002845866729815801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,64,0.0026528000831604003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,16,32,32,0.0026634665826956432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,7168,0.31432746251424154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,8192,0.3653813362121582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,10240,0.4514133453369141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,12288,0.5429226557413738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,6144,0.2710655848185221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,5120,0.22756373087565102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,4096,0.19848213195800782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3584,0.1885632038116455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,16384,0.7126794815063476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2560,0.13568320274353027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,3072,0.14136212666829426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,2048,0.09187093575795492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1024,0.05592853228251139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,1536,0.07205653190612793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,64,0.006265600025653839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,128,0.00886293351650238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,256,0.016691199938456216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,768,0.03746773401896159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,512,0.029816534121831256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,65536,32,0.004413866500059763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,8192,0.08889813423156738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,10240,0.11024426619211833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,12288,0.1343392054239909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,16384,0.17453120549519857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,7168,0.07854186693827311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,6144,0.06837973594665528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,5120,0.058251734574635824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,4096,0.05375359853108724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3584,0.04121066729227702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,3072,0.04002133210500081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2560,0.030550400416056316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,768,0.01120746632417043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,2048,0.025075199206670125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1024,0.014070399602254233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,1536,0.0199072003364563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,512,0.008348799745241801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,256,0.005634133517742157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,128,0.003454933315515518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,64,0.0031221332649389905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,32,0.0031370667119820913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,12288,0.10576000213623046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,10240,0.08484906355539958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,16384,0.13779519399007162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,7168,0.059638400872548424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,8192,0.06985599994659424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,6144,0.052484265963236486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,4096,0.03762133518854777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,5120,0.043828264872233076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3584,0.031941332419713336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,3072,0.02799466649691264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2048,0.02002560098965963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,2560,0.024178133408228556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1024,0.011436800161997478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,1536,0.017640533049901326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,768,0.009672533472379048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,512,0.007500799993673961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,256,0.0038346665600935614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,128,0.0035071998834609987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,64,0.003201066702604294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,32,0.00315733328461647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,16384,65536,0.7011914571126302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,10240,0.083133864402771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,12288,0.08622079690297445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,16384,0.1130677302678426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,7168,0.05156906843185425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,12288,65536,0.5358783721923828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,8192,0.058355200290679934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,6144,0.045620266596476236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3584,0.02746666669845581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,4096,0.03524693250656128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,5120,0.037929598490397134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,3072,0.024133332570393882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1536,0.01343786617120107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,1024,0.009963732957839967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2048,0.01695893406867981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,2560,0.020566399892171225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,768,0.008238933483759562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,512,0.0065749332308769224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,256,0.0035584000249703727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,128,0.003202133377393087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,64,0.002932266642649968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,32,0.0030037333567937215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,12288,0.06766719818115234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,10240,0.05667093197504679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,16384,0.0880191961924235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,7168,0.04078293244043986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,8192,0.04613120158513387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,6144,0.036200535297393796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,10240,65536,0.43457279205322263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,4096,0.024910932779312132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3584,0.02226026654243469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,5120,0.030507733424504597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,3072,0.022555732727050783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1536,0.011169067025184632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2560,0.016747732957204185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,2048,0.013990400234858194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,1024,0.008506666620572407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,768,0.007119999825954437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,256,0.0034389334420363107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,512,0.005658666789531708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,128,0.00325546662012736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,32,0.0029504001140594482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,64,0.00297173336148262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,10240,0.053831466039021815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,12288,0.06341546773910522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,16384,0.08144000371297201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,8192,65536,0.35415786107381186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,7168,0.0395146648089091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,8192,0.04425493478775024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,5120,0.033878401915232344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,6144,0.035240534941355386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,4096,0.02258666753768921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3584,0.020197333892186482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,3072,0.018017067511876424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2560,0.01746666630109151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,2048,0.012883200248082479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1536,0.010540800293286641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,1024,0.008012799918651581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,768,0.006742399930953979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,512,0.004103466620047887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,128,0.0030570665995279947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,256,0.003289599965016047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,64,0.0029418667157491045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,32,0.0028533334533373516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,10240,0.044377601146698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,7168,65536,0.30624214808146155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,12288,0.05539199908574423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,16384,0.07191359996795654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,8192,0.03646293481190999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,7168,0.037581864992777506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,5120,0.02472106615702311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,6144,0.029064534107844035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2560,0.014134400089581809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3584,0.018232532342274985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,4096,0.020338133970896403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,3072,0.01643946667512258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1536,0.009576533238093059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,2048,0.012032000223795573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,1024,0.007275733351707459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,768,0.006277333199977875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,512,0.003681066632270813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,128,0.003017599880695343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,256,0.0032479998966058097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,64,0.0029706666866938275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,32,0.0028245332340399425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,10240,0.03766080141067505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,12288,0.04448320070902507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,6144,65536,0.2648266633351644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,16384,0.056601599852244056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,8192,0.03072426716486613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,7168,0.027335466941197713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,5120,0.020444800456364952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,6144,0.0276202658812205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2560,0.011870933572451274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3072,0.013414399822553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,3584,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,4096,0.017172267039616905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,2048,0.010140800476074218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1536,0.008302933474381765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,1024,0.0067114666104316715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,768,0.005212800204753875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,512,0.0035466666022936503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,256,0.0032543999453385672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,128,0.00306986669699351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,64,0.0028234665592511495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,32,0.0028223998844623564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,5120,65536,0.22572479248046876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,12288,0.0386079986890157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,10240,0.03291413386662801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,16384,0.048786131540934245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,8192,0.030929066737492877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,7168,0.024705066283543905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,6144,0.022125866015752158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,5120,0.01947306593259176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,4096,0.01621333360671997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3584,0.012845866878827415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,3072,0.011497599879900615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2560,0.010141866405804952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,2048,0.008711466193199157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1024,0.005990399916966756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,1536,0.0074421331286430355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,768,0.003828266759713491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,512,0.0035232000052928926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,256,0.0031797334551811217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,128,0.0028309332827727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,64,0.002887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,32,0.0027818667391935987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,4096,65536,0.17941865921020508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,10240,0.02967039942741394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,12288,0.034203732013702394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,16384,0.043894398212432864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,8192,0.027893332640329997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,6144,0.019959467649459838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,5120,0.017578667402267455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,7168,0.022452267011006673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,4096,0.012902399897575379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3584,0.011845333377520244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,3072,0.010636799534161885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2560,0.009435733159383137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,2048,0.00824533353249232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1536,0.0070709332823753355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,1024,0.00479360024134318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,768,0.0038005332152048744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,512,0.0034954667091369627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,256,0.003201066702604294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,64,0.0028031999866167706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,32,0.002792533238728841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,128,0.002844800055027008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3584,65536,0.15691839853922526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,10240,0.026234666506449383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,8192,0.024884267648061117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,12288,0.03068373401959737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,16384,0.037843199570973714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,4096,0.013869866728782654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,5120,0.01579200029373169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,6144,0.017897599935531618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,7168,0.02002240022023519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3584,0.012824533383051553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2560,0.00881813367207845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,3072,0.011995733777681986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,2048,0.007565866907437642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1024,0.0044608001907666525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,1536,0.006614399949709575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,768,0.0037962667644023894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,512,0.003419733295838038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,128,0.0029738667110602063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,256,0.003138133386770884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,65536,0.1370207945505778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,32,0.002696533252795537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,3072,64,0.0027488000690937043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,10240,0.022899200518925987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,8192,0.019186133146286012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,12288,0.02982826630274455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,16384,0.03302293419837952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,6144,0.015505066514015198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,4096,0.01226026713848114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,5120,0.014020267128944396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,7168,0.017685333887736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3072,0.010654933253924052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,3584,0.01125866671403249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2560,0.008177066842714947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,2048,0.0070816000302632645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1536,0.005905066430568695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,1024,0.004080000023047129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,768,0.003755733370780945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,512,0.0034517332911491393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,65536,0.11576106548309326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,256,0.003094399968783061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,128,0.002922666569550832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,32,0.002739199995994568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2560,64,0.0028042666614055633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,16384,0.027218133211135864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,10240,0.019205333789189656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,8192,0.01603626708189646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,12288,0.02206826607386271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,7168,0.014749866724014283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,5120,0.01218666632970174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,6144,0.0132533331712087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,4096,0.010729599992434185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3584,0.0102101335922877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,3072,0.00950933297475179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2560,0.007467733323574066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,2048,0.006411733229955037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,65536,0.0920682668685913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1536,0.004686933259169261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,768,0.0036490666369597114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,1024,0.004021333406368891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,512,0.0033439998825391137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,256,0.0030591999491055804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,128,0.0029120000700155893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,64,0.002746666719516118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,2048,32,0.002799999962250392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,16384,0.022062933444976805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,10240,0.015675733486811318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,12288,0.01810986598332723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,8192,0.013753599921862283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,7168,0.012560000022252401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,6144,0.011479467153549194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,4096,0.009633066256841023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,5120,0.010726400216420491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3072,0.008664533495903015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,3584,0.00918293297290802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,65536,0.07209386825561523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2560,0.006807466844717662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,2048,0.005352533360322317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1024,0.004020266731580098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,1536,0.004705066482226053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,512,0.003401600072781245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,768,0.0037834666669368743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,256,0.0031167998909950255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,128,0.0028970666229724885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,64,0.002776533365249634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1536,32,0.0028031999866167706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,10240,0.012483200430870056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,16384,0.01649493376413981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,12288,0.01374613344669342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,8192,0.010986666878064473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,7168,0.01035093367099762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,6144,0.009351467092831928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,4096,0.008193066716194153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,5120,0.008992000420888265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,65536,0.048758399486541745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3584,0.007567999760309856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,3072,0.006970666845639547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2560,0.006066133578618368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,2048,0.005378133555253347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1536,0.004701866706212362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,1024,0.004020266731580098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,768,0.003700266778469086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,512,0.003387733300526937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,256,0.00306986669699351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,64,0.002713600049416224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,32,0.0027797333896160126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,1024,128,0.002872533351182938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,16384,0.013725866874059042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,12288,0.011539199948310852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,10240,0.011008000373840332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,8192,0.009845333298047383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,65536,0.03840426603953044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,7168,0.00922879974047343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,6144,0.008359466989835102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,5120,0.0077919999758402515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,4096,0.006715733309586842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3584,0.007062399884064992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2560,0.00598826656738917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,3072,0.006503466765085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,2048,0.005365333457787832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1536,0.004714666803677877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,1024,0.004033066580692927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,768,0.0036202666660149893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,512,0.003339733431736628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,256,0.0030165334542592366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,32,0.0027146667242050173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,128,0.002885333448648453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,768,64,0.0027456000447273255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,16384,0.011082667112350463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,12288,0.009597866733868917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,10240,0.009124267101287841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,8192,0.008154666423797608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,65536,0.028255999088287354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,7168,0.007210666437943776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,6144,0.006621866424878438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3072,0.006602666775385539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,5120,0.00695253312587738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,3584,0.0068896000583966565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,4096,0.006411733229955037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2560,0.006017066538333893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,2048,0.0053375999132792154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1536,0.004683733483155568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,1024,0.0039989332358042395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,768,0.003655466685692469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,512,0.003291733314593633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,256,0.0030720000465710956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,64,0.0027242665489514667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,128,0.0028917332490285236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,512,32,0.0027456000447273255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,16384,0.008191999793052674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,65536,0.01695573329925537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,12288,0.006729599833488464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,10240,0.006727466483910878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,8192,0.006675200164318084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,7168,0.006487466891606649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,6144,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,5120,0.006754133105278015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,4096,0.006295466423034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3072,0.006477866570154827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,3584,0.006833066542943318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2048,0.005323733389377594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,2560,0.006025599936644236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1024,0.004001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,1536,0.0046623999873797095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,768,0.003647999962170919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,512,0.0033226666351159418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,256,0.002995199958483378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,128,0.0028533334533373516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,64,0.0027093333502610523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,256,32,0.0027189334233601887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,65536,0.012282666563987733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,16384,0.006263466676076253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,12288,0.006301866471767425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,8192,0.006458666423956554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,10240,0.006494933366775512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,6144,0.006064000229040781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,7168,0.006320000191529592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,5120,0.00653546651204427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,4096,0.0060810665289560955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3584,0.006614399949709575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,3072,0.006213333209355672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2560,0.005701333284378052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,2048,0.005099733173847198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,768,0.0035189333061377203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1536,0.004479999840259552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,1024,0.003803733239571253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,512,0.003209600100914637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,256,0.0029909332593282064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,128,0.002808533360560735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,64,0.0027114666998386385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,128,32,0.0026549334327379864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,65536,0.009455999732017517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,16384,0.006313600142796834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,12288,0.006157866617043813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,10240,0.006381866832574208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,7168,0.006161066889762879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,8192,0.0062282666563987735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,6144,0.006088533500830332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,5120,0.006412800153096516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3584,0.006526933113733928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,4096,0.0060586666067441305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2560,0.005589333176612854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,3072,0.006123733520507812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,2048,0.0050122668345769245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1536,0.0044383997718493145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,1024,0.0038005332152048744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,768,0.0035018667578697203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,512,0.003155199935038885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,256,0.0029056000212828318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,128,0.002784000088771184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,64,0.002661333233118057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,64,32,0.002669866631428401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,65536,0.0074527998765309645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,16384,0.006054399907588959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,12288,0.006230400005976359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,7168,0.006076799829800924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,10240,0.006346666812896728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,8192,0.0062837332487106325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,6144,0.005993600189685822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,5120,0.006426666676998138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,4096,0.005930666625499725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2560,0.005592533449331919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3584,0.006486399968465169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,3072,0.006183466811974844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,2048,0.0050005331635475155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1024,0.0037087999284267426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,1536,0.0043722664316495265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,512,0.003189333279927572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,768,0.003517866631348928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,256,0.002856533229351044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,128,0.0027295999228954316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,64,0.0026389333109060925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,8,32,32,0.0026581334571043652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,7168,0.31445652643839517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,8192,0.36478827794392904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,10240,0.45042559305826824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,12288,0.5410634358723958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,6144,0.2708266576131185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,5120,0.22726720174153647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,4096,0.1979658603668213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,16384,0.7139861424763997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3584,0.18851946194966634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2048,0.0916927973429362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1024,0.05068693161010742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,2560,0.1352682590484619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,3072,0.1413610617319743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,1536,0.08283946514129639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,768,0.03810773293177287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,128,0.009975467125574749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,256,0.014550399780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,512,0.025748266776402788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,32,0.004404266675313314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,65536,64,0.0060810665289560955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,8192,0.08892693519592285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,10240,0.11047146320343018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,12288,0.13284160296122233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,16384,0.1758944034576416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,7168,0.07830080191294352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,6144,0.06841706434885661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,5120,0.05703466733296712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,4096,0.050519466400146484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3072,0.036644268035888675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,3584,0.04095146656036377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2560,0.03475093444188436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,768,0.011271466811498005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,2048,0.02491733431816101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1024,0.014062933127085366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,1536,0.019683200120925903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,512,0.008405333757400513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,256,0.005419733126958212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,64,0.003066666672627131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,128,0.0034944000343481696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,32,0.0031360000371932983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,10240,0.08852480252583822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,12288,0.10508053302764893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,16384,0.13697279294331868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,8192,0.0714357296625773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,7168,0.059292801221211756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,6144,0.052094932397206625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,4096,0.03558719952901204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,5120,0.04610453446706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3584,0.03193493286768596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,3072,0.0278165340423584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2560,0.024780799945195518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,2048,0.019744000832239785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1024,0.011513599753379821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,768,0.009595732887585957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,1536,0.01550933321317037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,512,0.007527466615041096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,256,0.004080000023047129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,128,0.0034901333351929987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,64,0.003189333279927572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,32,0.0031626666585604347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,16384,65536,0.7014389038085938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,12288,0.08560640017191569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,10240,0.08328639666239421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,16384,0.11315306822458904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,8192,0.05807040135065714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,7168,0.05135999917984009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,6144,0.04495999813079834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,12288,65536,0.528600533803304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,4096,0.030713599920272828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,5120,0.03766186634699504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3584,0.027399466435114546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,3072,0.027356799443562823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1536,0.013300266861915589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,1024,0.009806933005650838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2560,0.02044266661008199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,2048,0.016746666034062704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,768,0.008236800134181977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,512,0.006486399968465169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,256,0.0035562666753927866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,128,0.0031498665610949195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,32,0.0028938665986061097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,64,0.002921599894762039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,10240,0.05654933452606201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,12288,0.06745279630025228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,16384,0.0878613313039144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,10240,65536,0.43889811833699544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,8192,0.04601706663767497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,7168,0.04064533313115438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,6144,0.03596373399098714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,4096,0.024740266799926757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3584,0.022249599297841392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,5120,0.03503893216451009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,3072,0.01962239940961202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1536,0.011251200238863628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,1024,0.008518399794896443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2560,0.016743467251459757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,2048,0.013972266515096029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,768,0.007191466788450877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,256,0.0035434665779272715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,512,0.005445333321889242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,128,0.0032245332996050516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,32,0.0029824001093705496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,64,0.0030271999537944795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,10240,0.0531765341758728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,12288,0.06286613146464029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,16384,0.08245013554890951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,8192,65536,0.3545642534891764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,7168,0.039084800084431964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,8192,0.043816534678141277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,6144,0.034721068541208905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,5120,0.033613868554433185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,4096,0.022137600183486938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3584,0.020009599129358926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,3072,0.017758933703104655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2560,0.015345066785812378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,2048,0.01290773351987203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1536,0.010488532980283101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,1024,0.007941333452860515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,768,0.006748799979686737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,512,0.0042453333735466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,256,0.003489066660404205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,64,0.002919466545184453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,128,0.0030805334448814393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,32,0.0028586665789286296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,10240,0.04689600070317586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,7168,65536,0.3037280082702637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,12288,0.055002665519714354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,16384,0.07153706550598145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,7168,0.038372266292572024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,8192,0.03626559972763062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,6144,0.028845866521199543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,5120,0.025118933121363325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2560,0.014335999886194864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,4096,0.02060799996058146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3584,0.018065067132314046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,3072,0.016343466440836587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,2048,0.01214400033156077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1536,0.00965333382288615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,1024,0.007285333176453908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,768,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,512,0.0040618665516376495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,256,0.0033002667129039764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,128,0.003011200080315272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,64,0.002942933390537898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,32,0.002869333326816559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,10240,0.03758293390274048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,12288,0.044595201810201004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,16384,0.05810346603393555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,6144,65536,0.265555206934611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,5120,0.02362346649169922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,8192,0.0306442658106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,6144,0.02416320045789083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,7168,0.02760106722513835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3072,0.013567999998728434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,3584,0.015333333611488342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,4096,0.016999467213948568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2560,0.011961600184440613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,2048,0.010203733046849569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1536,0.008501332998275758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,1024,0.006679466863473256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,768,0.005268266797065735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,512,0.003664000084002813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,256,0.003205333401759466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,128,0.00308693324526151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,64,0.002811733384927114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,32,0.0028064000109831494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,5120,65536,0.2236032009124756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,10240,0.03255893389383952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,12288,0.038209064801534014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,16384,0.04842133522033691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,8192,0.030855466922124226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,7168,0.0244704008102417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,6144,0.02193066676457723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,4096,0.016107733050982155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,5120,0.019112533330917357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3584,0.012733866771062216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,3072,0.011590400338172912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2560,0.010194133718808491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,2048,0.008752000331878663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1536,0.007467733323574066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,768,0.003917866696914038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,1024,0.005701333284378052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,512,0.0034976000587145484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,256,0.003271466741959254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,128,0.002976000060637792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,64,0.00277866671482722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,32,0.002796799937884013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,4096,65536,0.17652160326639812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,10240,0.02935466567675273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,12288,0.034327467282613114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,8192,0.027754666407903035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,16384,0.04340693155924479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,4096,0.012956800063451132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,5120,0.017548799514770508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,6144,0.019843200842539467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,7168,0.022322134176890055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3584,0.011814399560292562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,3072,0.010620799660682679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2560,0.009423999985059103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,2048,0.008160000046094257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1536,0.0070271998643875126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,1024,0.004781866570313772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,768,0.0039711999396483105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,512,0.003509333233038584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,256,0.0031839999059836066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,128,0.0029877332349618276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,64,0.0027647999425729113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,65536,0.15573333104451498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3584,32,0.0028394666810830434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,12288,0.03028480013211568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,8192,0.021986132860183714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,16384,0.038736001650492353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,10240,0.02942933241526286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,5120,0.015715199708938598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,4096,0.013875200351079305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,7168,0.02003413240114848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,6144,0.017772799730300902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3584,0.012987732887268066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,3072,0.01165013313293457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2560,0.008775466680526733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,2048,0.007630933324495952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1024,0.004513066510359446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,1536,0.006587733328342438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,768,0.003798400113979975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,512,0.0035061334570248926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,256,0.0031360000371932983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,65536,0.13637439409891766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,128,0.0029728000362714132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,64,0.0027989332874615988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,3072,32,0.0027903998891512555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,8192,0.019272534052530925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,12288,0.02637866735458374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,10240,0.022855466604232787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,16384,0.03637973467508952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,4096,0.012184533476829528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,6144,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,5120,0.013957333564758301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,7168,0.017465599377950034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3584,0.0114656001329422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,3072,0.010614400108655293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2560,0.008142933249473572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,2048,0.007020799815654755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1536,0.00576853354771932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,512,0.0034677334129810332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,1024,0.004121600091457367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,768,0.0037962667644023894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,65536,0.11528426806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,256,0.003018666555484136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,128,0.002985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,64,0.0028031999866167706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2560,32,0.002775466690460841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,16384,0.026919466257095338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,8192,0.016008533040682473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,12288,0.02178666591644287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,10240,0.019013333320617675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,7168,0.014630400141080222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,6144,0.013329066832860312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,5120,0.012230400244394939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,4096,0.010796800255775452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3072,0.009634133179982502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,3584,0.010220799843470256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2560,0.0074976002176602675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,2048,0.006664533416430156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,65536,0.09154559771219889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1536,0.00469760000705719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,1024,0.004026666780312856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,768,0.0036992001036802924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,512,0.003386666625738144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,256,0.003052799900372823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,128,0.00288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,64,0.0027114666998386385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,2048,32,0.0027615999182065325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,10240,0.015656532843907674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,8192,0.013582932949066161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,16384,0.021826134125391642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,12288,0.017805866400400796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,7168,0.012493866682052612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,5120,0.010497066378593444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,6144,0.011688533425331115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,4096,0.009559466441472372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3584,0.009076266487439474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,3072,0.008611200253168742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,65536,0.07097386519114177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2560,0.006771199901898702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,2048,0.00556160012880961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1536,0.0046965335806210835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,1024,0.004019200056791306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,768,0.0036629334092140196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,512,0.003369600077470144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,128,0.0029077333708604175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,64,0.002682666728893916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,256,0.0030069333811601003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1536,32,0.0027434666951497394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,12288,0.013689600427945457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,10240,0.012149332960446676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,16384,0.016428800423940022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,8192,0.01090773344039917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,7168,0.010353066523869832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,5120,0.00897920032342275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,65536,0.048163199424743654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,6144,0.009339732925097148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,4096,0.008161066472530365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3584,0.007482666770617168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,3072,0.006885333359241486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2560,0.006135466694831848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,2048,0.0053493330876032506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1536,0.004680533210436503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,1024,0.004002133260170618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,768,0.003669333209594091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,512,0.0033834666013717652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,256,0.00297173336148262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,128,0.0028736000259717304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,64,0.0030016000072161358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,1024,32,0.002958933264017105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,10240,0.010917333761850993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,12288,0.011944533387819926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,16384,0.013901866475741067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,8192,0.00969493289788564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,7168,0.009218133489290873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,6144,0.008397866288820903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,5120,0.00796693315108617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,4096,0.006621866424878438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,65536,0.03789226611455281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3584,0.006984533369541168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,3072,0.006495999793211619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2048,0.005317333340644837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,2560,0.006016000111897787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1536,0.004690133531888326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,1024,0.004008533308903376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,512,0.0032757334411144257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,768,0.0036501333117485045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,256,0.0032618666688601174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,128,0.0029706666866938275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,32,0.0027978666126728057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,768,64,0.0028533334533373516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,12288,0.009713066617647807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,16384,0.01102293332417806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,10240,0.009154133001963298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,8192,0.008027733365694682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,65536,0.027769599358240766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,7168,0.0070783997575442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,6144,0.006783999999364217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,5120,0.0068256000677744556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,4096,0.00639573335647583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3584,0.006818133095900218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,3072,0.006506666541099548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2560,0.006107733150323232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,2048,0.005388799806435903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1536,0.004755199948946635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,768,0.0037802666425704955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,1024,0.004127999891837438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,256,0.0030741333961486817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,512,0.003472000112136205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,128,0.002902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,64,0.0028213332096735638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,512,32,0.00278613343834877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,16384,0.007843199868996937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,10240,0.006788266698519389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,12288,0.006674133241176605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,65536,0.01688213348388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,8192,0.00652266691128413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,6144,0.006296533346176148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,7168,0.006312533219655354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,5120,0.0066442668437957765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,4096,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3584,0.006725333134333293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,3072,0.006371200084686279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2048,0.005321600039800008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,2560,0.006005333364009857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1536,0.004682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,1024,0.004033066580692927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,768,0.0036533333361148832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,512,0.0033546666304270422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,256,0.003048533449570338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,128,0.0028746667007605235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,32,0.002752000093460083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,256,64,0.00276799996693929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,65536,0.011866666873296102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,16384,0.006262399752934774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,12288,0.006326400240262349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,10240,0.006523733337720235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,8192,0.006371200084686279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,7168,0.006277333199977875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,6144,0.006185600161552429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,5120,0.006435200075308482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,4096,0.006039466460545858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3584,0.006570666531721751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,3072,0.006267733375231425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2560,0.005634133517742157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,2048,0.005075199902057648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1536,0.00451093316078186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,1024,0.003868799904982249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,768,0.003636266787846883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,256,0.0030400000512599947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,512,0.003332266708215078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,32,0.0027402666707833606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,128,0.002869333326816559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,128,64,0.002746666719516118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,65536,0.009491200248400371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,16384,0.006111999849478403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,12288,0.006192000210285186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,10240,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,8192,0.006296533346176148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,5120,0.006479999919732411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,6144,0.006101333101590474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,7168,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3584,0.00653546651204427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,4096,0.006016000111897787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2560,0.00554666668176651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,3072,0.006189866860707601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,2048,0.005003733436266581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1536,0.004404266675313314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,1024,0.003772799919048945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,768,0.0034080001215140024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,512,0.0031583999594052637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,256,0.0029546665648619336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,128,0.002863999952872594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,64,0.0027850667635599775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,64,32,0.002666666607062022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,65536,0.008322133123874665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,16384,0.006083199878533682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,12288,0.006217599908510844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,10240,0.006301866471767425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,8192,0.006286933521429698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,7168,0.006168533364931742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,6144,0.006022400160630544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,4096,0.005987200140953064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3584,0.006410666803518932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,5120,0.006413866579532623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,3072,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2560,0.005580799778302511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,2048,0.004966400067011515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1536,0.004404266675313314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,1024,0.0037962667644023894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,768,0.003475199888149897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,256,0.0029290666182835894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,512,0.0031626666585604347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,128,0.0027477333943049112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,32,0.0026709333062171934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,4,32,64,0.0026730666557947796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,7168,0.3138474782307943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,8192,0.3616522789001465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,10240,0.4548927942911784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,12288,0.5412341435750325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,5120,0.22495786348978677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,6144,0.2704405466715495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,4096,0.198252805074056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3584,0.1883061408996582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,16384,0.7138346354166667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2560,0.1205514669418335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,3072,0.1462293306986491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,2048,0.09206720193227133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1024,0.05574186642964681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,1536,0.07139946619669596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,512,0.029478400945663452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,64,0.006098133325576782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,128,0.008869333068529765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,256,0.016453333695729575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,768,0.04296213388442993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,65536,32,0.004468266665935516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,8192,0.08868052959442138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,10240,0.1096010684967041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,12288,0.13370560010274252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,16384,0.17419519424438476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,7168,0.07805439631144205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,6144,0.06802133719126383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,5120,0.057348267237345374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,4096,0.054545064767201744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3584,0.04088213443756104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,3072,0.03977280060450236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2560,0.030158933003743487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,2048,0.024685867627461753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,768,0.011262933413187664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1024,0.013946666320164999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,1536,0.019556266069412232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,512,0.00844586690266927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,256,0.005686399837334951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,128,0.003442133218050003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,64,0.00308693324526151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,32,0.0031210665901501974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,10240,0.08433919747670492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,12288,0.10474773248036702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,16384,0.13629546165466308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,8192,0.06776639620463053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,7168,0.0632149338722229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,6144,0.05350399812062582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,4096,0.037777066230773926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,5120,0.04370666742324829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3584,0.03255040049552917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,3072,0.029287467400232952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2560,0.023619200785954794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,2048,0.019977599382400513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,768,0.009591466188430786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1024,0.011387733618418376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,1536,0.015691733360290526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,128,0.0034805332620938623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,256,0.004124800115823746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,512,0.0075445334116617845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,16384,65536,0.6999327977498372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,64,0.0031968000034491217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,32,0.0031839999059836066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,10240,0.07196906407674154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,12288,0.0861290693283081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,16384,0.13022826512654623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,8192,0.0580725351969401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,7168,0.051327999432881674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,12288,65536,0.528109868367513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,6144,0.04485013484954834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,4096,0.03064853350321452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,5120,0.03774720033009847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3072,0.023909332354863484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,3584,0.031137067079544067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2560,0.020402133464813232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,2048,0.01686613361040751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1536,0.013317333658536276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,1024,0.00981440047423045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,768,0.008278400202592214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,512,0.006557866434256236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,256,0.003668266783157984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,128,0.003171200056870779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,64,0.0030303999781608583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,32,0.00306986669699351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,10240,0.05626560052235922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,12288,0.06733653545379639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,16384,0.08942933082580566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,8192,0.04572266737620036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,7168,0.04056426684061686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,10240,65536,0.4434965451558431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,6144,0.035918935139973955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,4096,0.024647466341654458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3584,0.025620265801747637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,5120,0.030188800891240437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,3072,0.01958400011062622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1024,0.008474666873613994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,1536,0.011203199625015259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2560,0.016707199811935424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,2048,0.013827199737230936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,256,0.0035616000493367515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,768,0.007206400235493978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,512,0.005538133283456167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,128,0.0031445334355036415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,64,0.002985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,32,0.0029525332152843474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,10240,0.05288213491439819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,12288,0.0625983993212382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,16384,0.08047786553700766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,8192,65536,0.352837340037028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,5120,0.03366186618804932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,6144,0.03450666666030884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,8192,0.043567999203999834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,7168,0.03890986839930217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,4096,0.022216532627741495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2560,0.015299200018246969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3584,0.020066134134928384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,3072,0.01771413286526998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,2048,0.012892799576123557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1024,0.008949333429336548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,1536,0.01053013304869334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,768,0.006720000008742015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,512,0.004189866781234741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,256,0.0034442665676275887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,128,0.0030975999931494398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,64,0.002922666569550832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,32,0.002872533351182938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,10240,0.046380798021952316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,12288,0.05473706722259521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,16384,0.07123093605041504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,7168,65536,0.30471893946329753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,8192,0.03777600129445394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,7168,0.03354026476542155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,6144,0.028255999088287354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,5120,0.028691200415293376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2560,0.014351999759674073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,4096,0.020466132958730062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3072,0.015893333156903586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,3584,0.018733867009480796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1536,0.009641599655151368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,2048,0.012204800049463909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,1024,0.007267199953397115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,768,0.0062720000743865965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,512,0.003916800022125244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,256,0.0034058667719364167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,64,0.002899199972550074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,128,0.0030303999781608583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,32,0.0027978666126728057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,12288,0.044606932004292804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,10240,0.03707626660664876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,6144,65536,0.26471999486287434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,16384,0.05646506547927856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,7168,0.02696640094121297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,8192,0.030481066306432086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,5120,0.020282665888468422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,6144,0.027692800760269164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3584,0.015256533026695251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,3072,0.013460266590118408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2560,0.011867733796437581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,4096,0.016924800475438435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,2048,0.010121599833170573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1024,0.006664533416430156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,1536,0.008402132987976074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,768,0.005505066613356272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,512,0.0037269333998362223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,256,0.003294933338960012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,64,0.002796799937884013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,128,0.0030048000315825146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,32,0.002898133297761281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,5120,65536,0.22527999877929689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,12288,0.0383573333422343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,10240,0.032440533240636185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,16384,0.048163199424743654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,8192,0.030819199482599896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,6144,0.02167146603266398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,5120,0.01916159987449646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,7168,0.024387200673421226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,4096,0.016323199868202208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3584,0.012820266683896384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,3072,0.011553066968917846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2560,0.010207999746004741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,2048,0.008851200342178345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1536,0.00758186678091685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,1024,0.005932799975077311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,768,0.0038805333276589714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,512,0.0035616000493367515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,256,0.003180799881617228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,128,0.002958933264017105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,64,0.0027893332143624624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,32,0.002828799933195114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,4096,65536,0.17601067225138348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,12288,0.03413866758346558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,10240,0.029228800535202028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,16384,0.04302826722462972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,8192,0.027729066212972005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,4096,0.012897066275278726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,7168,0.02225173314412435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,5120,0.01730239987373352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,6144,0.01957546671231588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3584,0.011773866415023804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,3072,0.010602666934331258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2048,0.00827946662902832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,2560,0.009436800082524618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1536,0.007035733262697856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,1024,0.004587733248869578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,768,0.003881600002447764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,512,0.003538133452335993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,256,0.0032597333192825317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,128,0.0029813334345817565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,64,0.0027903998891512555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,65536,0.1551509380340576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3584,32,0.0027327999472618104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,8192,0.021578667561213176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,10240,0.029384533564249676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,12288,0.030137600501378377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,16384,0.03883519967397054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,5120,0.015640532970428465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,4096,0.014216533303260804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,7168,0.019621332486470543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,6144,0.01741973360379537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3584,0.013079466422398886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,3072,0.011813333630561829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2560,0.008795733253161114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,2048,0.007565866907437642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1024,0.004159999887148539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,1536,0.006634666522343953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,768,0.0038592000802357995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,512,0.003553066651026408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,256,0.003223466624816259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,128,0.002985599885384242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,64,0.0027562665442625684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,32,0.0027935999135176343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,3072,65536,0.13582293192545575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,10240,0.022768000761667885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,8192,0.021592533588409422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,16384,0.032762666543324784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,12288,0.02609813412030538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,6144,0.015719466408093772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,5120,0.013987200458844504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,4096,0.012237866719563801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,7168,0.017497599124908447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3072,0.010682666301727295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,3584,0.011434666315714518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2560,0.008166400094827015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,2048,0.007044266661008198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1536,0.005781333148479462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,1024,0.004161066561937332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,768,0.0038250667353471124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,512,0.0034656000634034475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,65536,0.1106602668762207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,256,0.003201066702604294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,128,0.0028970666229724885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,64,0.0027114666998386385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2560,32,0.0028053333361943563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,10240,0.0189194659392039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,12288,0.021709867318471274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,8192,0.016025599837303162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,16384,0.02690666715304057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,7168,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,5120,0.012236799796422322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,6144,0.013357866803805032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,4096,0.010921600461006164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3072,0.009665066997210186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2560,0.007422933479150136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,2048,0.006503466765085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,3584,0.010251733660697936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,65536,0.09227200349171957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1536,0.004727466901143392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,1024,0.004021333406368891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,768,0.003692800054947535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,512,0.0033013333876927694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,128,0.002902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,64,0.002733866622050603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,256,0.003068800022204717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,2048,32,0.0027615999182065325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,8192,0.013637333114941915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,10240,0.01579839984575907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,16384,0.021760000785191854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,12288,0.017835734287897746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,7168,0.01274773379166921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,6144,0.011574400464693706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,5120,0.0106495996316274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,4096,0.009598933657010396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3584,0.009075199564297993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,65536,0.0705087979634603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2560,0.0070165331164995836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,3072,0.008615466952323913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,2048,0.005352533360322317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1024,0.004045866678158442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,1536,0.00469760000705719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,768,0.003673599908749262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,512,0.003401600072781245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,256,0.0031082667410373688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,128,0.0028949332733949023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,64,0.0027818667391935987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1536,32,0.0026709333062171934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,8192,0.01095466713110606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,10240,0.012549333771069846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,12288,0.013704533378283182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,16384,0.016378666957219443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,7168,0.010290132959683736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,6144,0.009331199526786804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,5120,0.008924800157546996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,4096,0.008246399958928426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,65536,0.048479998111724855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3584,0.0076117331782976794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,3072,0.006808533271153767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2560,0.006016000111897787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,2048,0.005364266534646353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1024,0.004023466755946477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,1536,0.004664533336957296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,768,0.0036586667100588477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,512,0.003349333256483078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,128,0.0028949332733949023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,256,0.0031093334158261614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,64,0.0029866665601730345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,1024,32,0.0029525332152843474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,16384,0.01372053325176239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,12288,0.011897599697113037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,8192,0.009805867075920105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,10240,0.01088106632232666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,7168,0.009193600217501322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,6144,0.008354133367538452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,65536,0.03790080149968465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,5120,0.00783253312110901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,4096,0.006685866912206013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3584,0.007110400001207988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,3072,0.006504533191521962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2560,0.00602346658706665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,2048,0.005353599786758423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1536,0.004682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,1024,0.003982933362325033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,768,0.0036608000596364343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,512,0.003303466737270355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,256,0.003256533294916153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,64,0.002855466554562251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,128,0.0029728000362714132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,768,32,0.002867199977238973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,16384,0.011041067043940226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,12288,0.009636267026265462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,8192,0.008327466746171314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,10240,0.00909440020720164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,7168,0.007275733351707459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,65536,0.027356799443562823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,6144,0.0065994665026664736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,5120,0.006720000008742015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3584,0.0068245331446329755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,4096,0.006305066744486491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,3072,0.006382933259010315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2560,0.006118399898211161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,2048,0.005463466544946035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1536,0.004774400095144907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,1024,0.00409706657131513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,768,0.0037546666959921518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,512,0.0034730667869249977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,256,0.003134933362404505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,128,0.0029450667401154833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,32,0.0027615999182065325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,512,64,0.0027583998938401537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,16384,0.007946666578451793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,12288,0.006843733290831248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,65536,0.01687039931615194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,10240,0.006750933329264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,8192,0.006570666531721751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,7168,0.00639466643333435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,6144,0.006337066491444905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,5120,0.0066431999206542965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,4096,0.006167466441790262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3072,0.0064629331231117245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,3584,0.006729599833488464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2560,0.006028800209363302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,2048,0.005348266661167144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1536,0.0046847999095916745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,1024,0.004026666780312856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,768,0.0036650667587916053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,512,0.003345066557327906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,128,0.0028512001037597655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,256,0.0030741333961486817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,64,0.002705066651105881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,256,32,0.002712533374627431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,65536,0.011973333358764649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,16384,0.006133333345254262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,12288,0.006243200103441874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,10240,0.006436266501744588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,7168,0.006251733501752217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,5120,0.00647680014371872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,8192,0.006380799909432728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,6144,0.006060799956321717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,4096,0.006087466577688853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3584,0.0065194666385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,3072,0.006281599899133046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2048,0.005042133231957754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,2560,0.0056618665655454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1536,0.004506666461626689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,1024,0.0038346665600935614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,768,0.003634133438269297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,128,0.002868266652027766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,512,0.0032768001159032187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,256,0.0030432000756263735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,64,0.0027615999182065325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,128,32,0.0027242665489514667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,65536,0.009181867043177288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,12288,0.006153599917888641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,16384,0.006071466704209646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,10240,0.0063274666666984555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,7168,0.006145066519578298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,8192,0.006305066744486491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,6144,0.006110933423042297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,5120,0.006365866462389629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,4096,0.006041599810123444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2560,0.005610666672388713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3584,0.0065098668138186145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,3072,0.006214400132497152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,2048,0.004969599843025208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1536,0.004353066782156626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,1024,0.0037098666032155357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,512,0.0031445334355036415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,768,0.0034944000343481696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,128,0.002902399996916453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,256,0.003011200080315272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,64,0.0027327999472618104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,64,32,0.002735999971628189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,65536,0.008259200056393941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,16384,0.006265600025653839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,12288,0.006064000229040781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,10240,0.0063178668419520065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,8192,0.006268799801667531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,6144,0.0060245335102081295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,7168,0.006137600044409434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,5120,0.0063274666666984555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,4096,0.005915733178456625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3584,0.006446933249632518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,3072,0.006118399898211161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2560,0.005593599875768026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1536,0.004369066655635833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,2048,0.005005866785844167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,1024,0.0037952000896135964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,768,0.003475199888149897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,256,0.00290133332212766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,512,0.003260799994071325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,64,0.002678400029738744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,128,0.0027327999472618104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,2,32,32,0.0026933332284291584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,7168,0.31326932907104493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,8192,0.3603370666503906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,10240,0.44980586369832354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,12288,0.5405279795328777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,5120,0.2240938663482666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,6144,0.2699530601501465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,4096,0.19614292780558268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3584,0.18825173377990723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,16384,0.7194389343261719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2560,0.1202239990234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,3072,0.14562452634175618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,2048,0.09116693337758383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,256,0.016420267025629678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,768,0.04322346846262614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,512,0.029395200808842975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1024,0.05553493499755859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,128,0.00888320008913676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,64,0.006152533491452535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,1536,0.07117226918538412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,65536,32,0.0044064000248909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,8192,0.08894826571146647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,10240,0.10964266459147136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,12288,0.13233706951141358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,16384,0.17422399520874024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,7168,0.07803946336110433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,6144,0.06799999872843424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,5120,0.056676268577575684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,4096,0.05375253359476725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3584,0.04076053301493327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2560,0.030024532477060956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,3072,0.039596800009409586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,2048,0.024604799350102742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,768,0.011198932925860088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1024,0.01390720009803772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,1536,0.019458132982254028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,512,0.00842133363087972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,256,0.005584000051021576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,128,0.003432533393303553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,64,0.0030720000465710956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,32,0.003125333289305369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,10240,0.08949653307596842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,12288,0.10449493726094564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,16384,0.13632960319519044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,8192,0.07339413166046142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,7168,0.05951573451360067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,6144,0.05330986579259237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,4096,0.03554133176803589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,5120,0.046315733591715494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3584,0.03177066644032796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,3072,0.027438932657241823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2560,0.024829866488774617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,2048,0.019876267512639365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1024,0.011531733473141988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,768,0.009781333804130554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,1536,0.01536853313446045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,512,0.007584000130494435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,256,0.004016000032424927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,128,0.003398400048414866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,64,0.003218133250872294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,32,0.003222399950027466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,16384,65536,0.7085525512695312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,12288,0.0860970656077067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,10240,0.08224746386210123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,16384,0.11312426726023357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,12288,65536,0.5224287986755372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,8192,0.05798293352127075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,6144,0.04489386479059855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,7168,0.04961173137029012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,4096,0.03060373266537984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,5120,0.04340373277664185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3072,0.023439999421437582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,3584,0.027340799570083618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1536,0.01325759987036387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,1024,0.009948800007502239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2560,0.020323199033737183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,2048,0.0168938676516215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,768,0.00814933329820633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,512,0.0065536002318064375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,256,0.0036490666369597114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,128,0.00325546662012736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,64,0.002962133288383484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,32,0.002994133283694585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,10240,0.05631999969482422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,12288,0.0679637352625529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,16384,0.08824000358581544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,10240,65536,0.43509225845336913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,8192,0.04598293304443359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,7168,0.04079999923706055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,6144,0.03577386538187663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,4096,0.024578134218851723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,5120,0.035179734230041504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3072,0.019377066691716512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,3584,0.02221440076828003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1536,0.011218133568763732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,1024,0.008533333738644917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2048,0.013878400127092997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,2560,0.016683733463287352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,768,0.007133866846561432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,512,0.0056426664193471275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,256,0.003554133325815201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,128,0.0031466667850812277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,64,0.0029824001093705496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,32,0.0029535998900731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,10240,0.05273386637369791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,12288,0.06249493360519409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,16384,0.08054506778717041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,8192,65536,0.3538954734802246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,8192,0.04326293468475342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,7168,0.03866560061772664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,6144,0.03419839938481649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,5120,0.03357119957605998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3072,0.01768640081087748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,3584,0.01998293399810791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,4096,0.022325332959493002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2560,0.015267200271288552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,2048,0.012826666235923767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,768,0.006780800223350525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1536,0.010542933146158855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,1024,0.007892266909281413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,512,0.004310399790604909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,256,0.0034815999368826545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,128,0.0031818665564060213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,64,0.002919466545184453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,32,0.0029386666913827257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,10240,0.049058131376902264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,12288,0.05454186598459879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,7168,65536,0.304635747273763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,16384,0.07078080177307129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,8192,0.03629120190938313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,6144,0.03360639810562134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,7168,0.03375039895375569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,5120,0.02510720094045003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2560,0.014435199896494546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,4096,0.02005866765975952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3584,0.018361600240071614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,3072,0.016434133052825928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,2048,0.012152533729871113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1536,0.009524266918500264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,768,0.006275199850400289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,1024,0.007421866556008657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,512,0.004089600096146265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,256,0.003332266708215078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,128,0.0030400000512599947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,64,0.002844800055027008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,32,0.002856533229351044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,6144,65536,0.2658133347829183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,10240,0.0377621332804362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,12288,0.043995734055836996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,16384,0.0562666654586792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,6144,0.024075732628504435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,7168,0.02723520000775655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,8192,0.035436801115671795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,5120,0.020282665888468422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2560,0.011778133114178975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3072,0.013644799590110779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,3584,0.015292800466219583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,4096,0.01688746611277262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1536,0.008561066786448161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,2048,0.009959466258684794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,1024,0.0066890666882197065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,768,0.005471999943256378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,512,0.003737599899371465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,256,0.0032448001205921174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,128,0.0030229332546393077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,64,0.002850133428970973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,32,0.0027935999135176343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,5120,65536,0.22494293848673502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,10240,0.03237013419469197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,12288,0.037956265608469646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,16384,0.048190931479136154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,8192,0.030825599034627275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,7168,0.024289067586263022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,6144,0.021828265984853108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,5120,0.01916159987449646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,4096,0.01611306667327881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3584,0.01288640002409617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2560,0.010222933689753215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,3072,0.011587199568748475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,2048,0.008931199709574383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1024,0.005758933226267497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,1536,0.007589333256085714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,768,0.003982933362325033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,512,0.003573333223660787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,256,0.003155199935038885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,128,0.002998399982849757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,64,0.0027903998891512555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,32,0.0028277332584063213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,4096,65536,0.17801599502563475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,16384,0.04298880100250244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,12288,0.033853868643442794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,10240,0.029051733016967774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,8192,0.02770133415857951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,4096,0.012893866499265036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,7168,0.021947733561197915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,5120,0.01730666756629944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,6144,0.01944640080134074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3584,0.011789866288503011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,3072,0.010630399982134501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2560,0.009477333227793375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,2048,0.008198399841785432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1536,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,1024,0.0045621335506439206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,768,0.0039007999002933502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,512,0.0035135999321937563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,128,0.002974933385848999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,256,0.003143466760714849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,64,0.0027583998938401537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,65536,0.15518933931986492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3584,32,0.00276053324341774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,12288,0.02956266601880391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,8192,0.021491199731826782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,10240,0.02962133288383484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,16384,0.03849706649780273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,5120,0.015562666455904641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,7168,0.01959999998410543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,4096,0.014239999651908874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,6144,0.017385600010553996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3584,0.012948266665140786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,3072,0.01153706709543864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2048,0.007525333265463512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,2560,0.008764800429344178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1536,0.006611200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,1024,0.004263466596603394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,768,0.003886933376391729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,512,0.003542399903138479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,256,0.003219199925661087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,65536,0.1355178674062093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,128,0.00288426677385966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,64,0.0027434666951497394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,3072,32,0.002820266783237457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,12288,0.025891200701395674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,10240,0.022341332832972207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,8192,0.018869332472483315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,16384,0.036645332972208663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,4096,0.012155733505884806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,6144,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,5120,0.013870933651924133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,7168,0.01738986571629842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3584,0.011356799801190694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,3072,0.010532266894976298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2560,0.008177066842714947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,2048,0.007057066758473713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1536,0.005679999788602194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,512,0.0034783999125162757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,1024,0.004187733431657155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,768,0.003764266769091288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,65536,0.11514346599578858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,256,0.0032511999209721885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,128,0.002887466549873352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,64,0.0027114666998386385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2560,32,0.002765866617361705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,16384,0.026924800872802735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,12288,0.021657600005467733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,8192,0.016105600198109946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,10240,0.01888426740964254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,7168,0.014579199751218162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,6144,0.013286399841308593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,4096,0.01090773344039917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,5120,0.01220373312632243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3584,0.01011199951171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,3072,0.009588266412417095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,65536,0.09094613393147787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2048,0.006632533172766368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,2560,0.007478400071461995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1536,0.004717866579691568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,1024,0.004007466634114583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,768,0.003740799923737844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,512,0.0033781332274278007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,256,0.0030026666820049284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,128,0.002918400118748347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,64,0.0027253332237402597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,2048,32,0.0027776000400384264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,10240,0.01560426652431488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,12288,0.01785279909769694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,8192,0.01367573340733846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,16384,0.021868799130121866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,7168,0.01269760032494863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,6144,0.011622400085131327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,5120,0.010593066612879436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,4096,0.009538132945696514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3584,0.009044266740481059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,3072,0.008539733290672303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2048,0.005369600156943003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,2560,0.006981333096822103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,65536,0.07121280034383139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1536,0.0046858668327331545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,1024,0.004019200056791306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,768,0.003664000084002813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,512,0.0034005333979924522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,256,0.0030517332255840302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,128,0.002883200099070867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,64,0.002746666719516118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1536,32,0.002678400029738744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,12288,0.013715199629465737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,10240,0.012572800119717916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,8192,0.010998400052388509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,16384,0.01625920037428538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,7168,0.010213333368301391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,5120,0.008890666564305623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,6144,0.009371733665466309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,4096,0.008137600123882293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,65536,0.047993600368499756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3584,0.0074527998765309645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,3072,0.006796800096829732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2560,0.006018133461475372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,2048,0.005334400137265523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1024,0.004001066585381826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,1536,0.004692266881465912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,768,0.0036159999668598174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,512,0.003369600077470144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,128,0.0028789333999156954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,256,0.002995199958483378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,32,0.002974933385848999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,1024,64,0.0030346666773160298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,16384,0.013800533612569174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,12288,0.011814399560292562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,10240,0.010870400071144103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,8192,0.00988159974416097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,65536,0.03779199918111165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,7168,0.009153067072232565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,6144,0.008330666522185007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,5120,0.007884799937407176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,4096,0.006588799754778545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3584,0.006829866766929626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2560,0.006002133091290792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,3072,0.006469333171844482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,2048,0.005321600039800008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1024,0.003995733211437861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,1536,0.0046847999095916745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,768,0.0036298667391141256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,256,0.00310506671667099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,512,0.003337600082159042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,128,0.003048533449570338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,64,0.002872533351182938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,768,32,0.002898133297761281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,16384,0.01097920040289561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,12288,0.009707732995351156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,65536,0.02728533347447713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,10240,0.009169066945711773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,8192,0.008078933258851369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,7168,0.007445333401362102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,6144,0.006651733318964641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,5120,0.006759466727574666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,4096,0.006378666559855144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3584,0.006794666747252147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,3072,0.006442666550477346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2560,0.006082133452097575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,2048,0.005410133302211762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1536,0.004733866453170777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,1024,0.0041514667371908825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,768,0.0037685332198937735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,512,0.0034389334420363107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,256,0.0031306666632493338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,128,0.0029056000212828318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,64,0.0028053333361943563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,512,32,0.0027744000156720476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,16384,0.008011733492215473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,65536,0.017010132471720375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,12288,0.00674773355325063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,10240,0.006593066453933716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,8192,0.006528000036875407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,7168,0.006380799909432728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,6144,0.0063360000650088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,5120,0.006704000135262807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,4096,0.006237866481145223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3584,0.006696533163388569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,3072,0.006342400113741558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2560,0.006043733159701029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,2048,0.005336533486843109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1024,0.004008533308903376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,768,0.003639466563860575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,1536,0.004671999812126159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,512,0.003389866650104523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,256,0.0031082667410373688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,128,0.0028778667251269023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,64,0.002700799951950709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,256,32,0.002705066651105881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,65536,0.011988266309102377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,16384,0.0063391998410224915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,12288,0.0063498665889104204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,10240,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,7168,0.006236800054709116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,8192,0.006438399851322174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,5120,0.0064533332983652755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,6144,0.006157866617043813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,4096,0.006045866509278615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3584,0.006534400085608165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,3072,0.006159999966621399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2560,0.0056991999348004665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,2048,0.005115733544031779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1024,0.0038399999340375268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,1536,0.0044608001907666525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,768,0.003621333340803782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,512,0.003332266708215078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,256,0.003078400095303853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,128,0.002898133297761281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,64,0.002726399898529053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,128,32,0.0026975999275843303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,65536,0.009169066945711773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,16384,0.006279466549555461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,12288,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,8192,0.006340266764163971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,10240,0.006295466423034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,7168,0.006198399762312571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,5120,0.0064064001043637585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,6144,0.006060799956321717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,4096,0.005926399926344554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3072,0.006187733511130015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,3584,0.006461866696675618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2560,0.005550933380921682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,2048,0.004982399940490723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1536,0.004378666480382284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,1024,0.0038015998899936674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,768,0.0034080001215140024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,512,0.003186133255561193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,256,0.0030645333230495454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,128,0.002844800055027008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,64,0.0026848000784715016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,65536,0.008197333415349324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,64,32,0.002664533257484436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,10240,0.006367999811967213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,12288,0.006128000219662985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,16384,0.00626453310251236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,8192,0.006275199850400289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,7168,0.006086400151252747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,6144,0.006067200005054474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,5120,0.0063381334145863845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,4096,0.005994666616121928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3584,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,3072,0.006062933305899302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2560,0.005578666428724925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1024,0.003772799919048945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,2048,0.004970666766166687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,1536,0.004422399898370107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,512,0.0032597333192825317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,768,0.003425066669782003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,256,0.0029450667401154833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,128,0.0028064000109831494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,64,0.0026933332284291584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,float16,1,32,32,0.0026602665583292644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,7168,5.538581339518229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,8192,6.4207763671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,10240,8.002078755696614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,12288,9.45746561686198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,6144,4.678762817382813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,5120,4.0347658793131504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,4096,3.1858187357584637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,16384,12.847196451822915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3584,2.7943359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2560,2.068023427327474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,3072,2.4323582967122395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,2048,1.6332608540852864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1024,0.9256117502848307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,1536,1.2849183400472006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,768,0.7660746892293294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,256,0.45948158899943037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,128,0.4536757469177246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,512,0.5852874755859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,64,0.4360362688700358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,65536,32,0.4224960009256999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,8192,1.6959658304850262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,10240,2.1717013041178386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,12288,2.5723808288574217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,16384,3.4434687296549478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,7168,1.4861323038736978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,6144,1.3044811248779298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,5120,1.0914549509684244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,4096,0.8737898508707682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3072,0.6811434427897136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,3584,0.7708970387776692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2560,0.5701610565185546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,768,0.22124053637186686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1024,0.2683338801066081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,1536,0.3699541409810384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,2048,0.4642666816711426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,256,0.13479679425557453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,512,0.1652181307474772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,128,0.12377386887868244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,64,0.12026773293813069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,32,0.11839146614074707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,10240,1.6512138366699218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,12288,1.9524511973063152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,16384,2.649578603108724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,8192,1.317676798502604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,7168,1.1456778208414713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,6144,1.011617088317871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,4096,0.6724863688151042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,5120,0.8331733067830405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3584,0.605177625020345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2560,0.4397802670796712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,3072,0.5157471974690755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,2048,0.3672053337097168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1536,0.2852021217346191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,1024,0.20835092862447105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,768,0.17333119710286457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,128,0.09606293042500814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,256,0.10737706820170086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,512,0.13390506108601888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,64,0.0955456018447876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,32,0.0939466635386149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,16384,65536,14.002458699544272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,12288,1.7109130859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,16384,2.3255584716796873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,10240,1.4282005310058594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,8192,1.1523125966389975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,7168,1.0286975860595704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,5120,0.7345482508341472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,6144,0.8748330434163412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,4096,0.6038527806599935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3072,0.4559712092081706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,12288,65536,10.641517130533854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,3584,0.5227765401204427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2560,0.3924021402994792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1024,0.1867861270904541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,1536,0.2577909310658773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,2048,0.3196000099182129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,768,0.1576181411743164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,256,0.09376640319824218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,128,0.08368319670359294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,512,0.12373653252919514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,64,0.08238399823506673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,32,0.08158186276753744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,10240,1.221288553873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,12288,1.4869183858235677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,16384,1.9408149719238281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,8192,0.9665514628092448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,7168,0.8586624145507813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,6144,0.728872553507487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,4096,0.48623361587524416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,5120,0.6142655690511067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3584,0.42166932423909503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,10240,65536,9.312943522135416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2560,0.30660266876220704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,3072,0.3688938776652018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,2048,0.24836479822794594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,768,0.1297439972559611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1024,0.15613333384195965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,512,0.0985578695933024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,1536,0.21532053947448732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,128,0.07061546643575033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,64,0.07100906372070312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,256,0.07902506987253824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,32,0.0697546641031901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,10240,1.0928874969482423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,12288,1.3249237060546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,16384,1.7290388743082683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,8192,0.8800352096557618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,7168,0.7801610946655273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,6144,0.6648192087809245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,4096,0.45651092529296877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,5120,0.5591903686523437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3584,0.4004543940226237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,3072,0.3511530558268229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2560,0.2987317403157552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,2048,0.24629119237264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1536,0.19507519404093426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,1024,0.14408960342407226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,8192,65536,7.973783365885417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,768,0.11863573392232259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,256,0.0727189302444458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,512,0.09141546885172526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,64,0.06460906664530436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,128,0.06965440114339193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,32,0.06376746495564779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,10240,0.9949301401774088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,12288,1.1893418629964194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,16384,1.576869328816732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,8192,0.8029216130574545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,7168,0.7037194569905598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,6144,0.6067616144816081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,5120,0.5115914662679036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,4096,0.41662187576293946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3584,0.36775360107421873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,3072,0.3201216061909994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2560,0.27376960118611654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,2048,0.22626986503601074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,7168,65536,7.116753133138021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1024,0.13025493621826173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,1536,0.17894934018452963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,768,0.10696213245391846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,256,0.0662282665570577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,512,0.08472533226013183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,128,0.05807360013326009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,64,0.05813759962717692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,32,0.05738879839579264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,10240,0.8724767684936523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,12288,1.0413471857706704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,16384,1.379878362019857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,8192,0.7033440272013347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,7168,0.6182506561279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,6144,0.5376181284586589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,4096,0.36801919937133787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,5120,0.4520405451456706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3584,0.32823146184285484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,3072,0.2835306803385417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2560,0.24147733052571616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,2048,0.20213120778401694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,6144,65536,6.480715942382813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1024,0.11612586975097657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,1536,0.1577877362569173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,768,0.09855466683705648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,512,0.08090240160624186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,128,0.051528533299764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,256,0.05898559888203939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,64,0.05166506767272949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,32,0.051478401819864905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,10240,0.7681962966918945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,12288,0.9080181121826172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,16384,1.1962122599283853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,8192,0.617022959391276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,7168,0.5383829116821289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,6144,0.46529172261555984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,4096,0.32095680236816404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,5120,0.39711147944132485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3584,0.2835029284159342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2560,0.21372906366984049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,3072,0.2487114588419596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,2048,0.1757919947306315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,5120,65536,5.633507283528646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1024,0.10116053422292073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,1536,0.13902293841044108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,768,0.08414506912231445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,512,0.06876266797383626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,256,0.05187306801478068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,128,0.044707198937733963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,64,0.04489173491795857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,32,0.045348266760508224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,10240,0.7094666798909505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,12288,0.8379082361857095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,16384,1.1236661275227866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,8192,0.5710709253946941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,7168,0.49971946080525714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,6144,0.43487361272176106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,5120,0.3691391944885254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,4096,0.29907414118448894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3584,0.2657674630482992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,3072,0.23344106674194337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2560,0.19872213999430338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,2048,0.16446080207824706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,4096,65536,4.81064198811849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1024,0.09497706890106201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,1536,0.12966933250427246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,768,0.079148801167806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,512,0.0635253349939982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,256,0.048376532395680745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,128,0.04286613464355469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,32,0.04274880091349284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,64,0.0421120007832845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,10240,0.6513919830322266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,12288,0.7756032307942708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,16384,1.0307455698649088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,8192,0.5263680140177409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,7168,0.4624320030212402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,6144,0.40126825968424484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,5120,0.3396650632222494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,4096,0.2778357187906901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3584,0.24565653800964354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,3072,0.2160202662150065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2560,0.18479572931925456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,2048,0.1529098669687907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3584,65536,4.452730814615885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1024,0.08872640132904053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,1536,0.1197503964106242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,512,0.05921279986699422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,768,0.07364373207092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,256,0.044276265303293864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,128,0.03947306474049886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,64,0.039228800932566324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,32,0.039280001322428384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,10240,0.5966911951700846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,12288,0.71472962697347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,16384,0.9446677525838216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,8192,0.48299840291341145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,7168,0.4290421485900879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,6144,0.37040961583455406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,5120,0.3132704099019369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,4096,0.25858559608459475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3584,0.22812906901041666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2560,0.17194132804870604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,3072,0.19976320266723632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,2048,0.1411904017130534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,3072,65536,4.070939636230468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1024,0.08217173417409261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,768,0.06832640171051026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,1536,0.11232960224151611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,512,0.0555295983950297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,128,0.03516053358713786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,256,0.04090346495310466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,64,0.03459946711858113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,32,0.03460906744003296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,10240,0.5463008244832357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,12288,0.6509152094523112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,16384,0.8577920277913412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,8192,0.4415882746378581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,7168,0.39119466145833337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,6144,0.33654079437255857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,4096,0.23683093388875326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,5120,0.2871061325073242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3584,0.20773760477701822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2560,0.15665705998738605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,3072,0.18269440333048503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2560,65536,3.7170954386393227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,2048,0.1288810650507609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1536,0.10261867046356202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,1024,0.07535573641459146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,512,0.05025279919306437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,768,0.06318506797154745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,128,0.03138346672058105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,256,0.037834668159484865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,64,0.030139732360839843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,32,0.030343466997146608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,10240,0.49227094650268555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,12288,0.5832906723022461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,16384,0.7706229527791341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,8192,0.39778025945027673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,7168,0.35042667388916016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,6144,0.3050485293070475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,4096,0.21071786880493165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,5120,0.2589248021443685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3584,0.18851733207702637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2560,0.1418282667795817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,3072,0.16387306849161784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,2048,65536,3.4175211588541665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,2048,0.11792213122049969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1024,0.06816213130950928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,1536,0.0931488037109375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,768,0.057226665814717616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,512,0.04475626548131307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,128,0.02992853323618571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,256,0.033212800820668534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,32,0.027191466093063353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,64,0.029098665714263915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,10240,0.4384607950846354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,12288,0.5192266782124837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,16384,0.6863391876220704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,8192,0.3535317420959473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,7168,0.31027841567993164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,6144,0.26789652506510414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,5120,0.22605652809143068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,4096,0.1857866605122884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3584,0.16465387344360352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2560,0.12516586780548095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,3072,0.14543253580729168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1536,65536,3.042809549967448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,2048,0.10415893395741779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,512,0.03476053476333618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1024,0.05819946527481079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,1536,0.0811797300974528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,768,0.04636160135269165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,256,0.02592746615409851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,128,0.022660267353057862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,32,0.021499733130137123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,64,0.02131839990615845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,10240,0.40914347966512044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,12288,0.49170347849527996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,16384,0.646994145711263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,8192,0.33018452326456704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,7168,0.2931104024251302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,6144,0.2522165298461914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,4096,0.17708479563395182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,5120,0.2141045411427816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3584,0.1560981273651123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,1024,65536,2.7096214294433594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2560,0.11817599932352703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,3072,0.13797440528869628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,768,0.040626132488250734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,2048,0.09766720136006674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1024,0.051830399036407473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,1536,0.07573333581288656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,64,0.02031573255856832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,512,0.03265066742897034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,128,0.02104533314704895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,256,0.023798400163650514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,32,0.020003199577331543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,10240,0.382154655456543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,12288,0.45410668055216474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,16384,0.6011199951171875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,8192,0.309713077545166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,7168,0.2715797424316406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,6144,0.23576107025146484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,4096,0.16339839299519857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,5120,0.20058132807413737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3584,0.1459925333658854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,768,65536,2.5542933146158857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2560,0.1102410634358724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,3072,0.12823253472646076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,2048,0.09074666500091552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1024,0.0456063985824585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,768,0.03591466744740804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,1536,0.06926613648732503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,64,0.018734933932622273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,256,0.022562134265899658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,128,0.019169066349665323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,512,0.027841067314147948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,32,0.018553600708643595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,10240,0.36415786743164064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,12288,0.43007678985595704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,16384,0.5664127985636394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,8192,0.2939530690511068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,7168,0.2586954593658447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,6144,0.22393600145975748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,4096,0.155238405863444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,5120,0.1907968044281006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3584,0.13746879895528158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,512,65536,2.3796597798665364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2560,0.10344853401184081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,3072,0.12048426469167073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,2048,0.08382293383280436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1024,0.03988373279571533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,768,0.03195733428001404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,1536,0.06231679916381836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,512,0.026324266195297243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,256,0.021128533283869426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,64,0.017461333672205606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,32,0.017046399911244712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,128,0.01806079943974813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,10240,0.35765441258748376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,12288,0.4258570671081543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,16384,0.5653226852416993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,8192,0.28887999852498375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,7168,0.25423787434895834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,6144,0.22176853815714517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,4096,0.15142399470011395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,5120,0.18592747052510578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3584,0.13533120155334472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,256,65536,2.232838439941406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2560,0.09999787012736003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,3072,0.11695466836293537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,768,0.030376533667246502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,2048,0.08046186765034993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1024,0.03691306511561076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,1536,0.05805439949035644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,128,0.01730453372001648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,256,0.019986132780710854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,64,0.01649493376413981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,512,0.025195733706156416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,32,0.01644159952799479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,10240,0.3561610539754232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,12288,0.42447573343912764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,16384,0.5623989105224609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,7168,0.25333120028177897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,8192,0.2875221252441406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,6144,0.22068373362223306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,4096,0.15107520421346027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,5120,0.18521812756856282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3584,0.13488319714864094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,128,65536,2.229107157389323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,3072,0.11680746873219808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2560,0.0984831968943278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,2048,0.08004480202992757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1536,0.05773760080337524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,768,0.03026026686032613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,1024,0.03642666737238566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,512,0.02509760061899821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,256,0.01996906598409017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,128,0.017313067118326822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,64,0.01616213321685791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,32,0.016178133090337117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,10240,0.3565578778584798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,12288,0.4226933479309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,16384,0.5581482569376628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,8192,0.28820266723632815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,7168,0.2532927989959717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,6144,0.21905919710795083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,5120,0.18562347094217937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,4096,0.15038506189982098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3584,0.13289600213368732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,64,65536,2.2205759684244795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,3072,0.11642133394877116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2560,0.09814080397288004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,2048,0.07801173528035482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1024,0.03482986688613891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,1536,0.055985065301259365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,768,0.030019199848175047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,512,0.02494293252627055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,256,0.01956693331400553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,64,0.016033066312472026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,128,0.016921599706014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,32,0.015612799922625223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,8192,3.038228352864583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,10240,3.691681162516276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,12288,4.579430643717448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8192,32,65536,2.2160886128743487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,16384,6.286471557617188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,7168,2.6537129720052084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,6144,2.3204981486002607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,5120,1.9524341583251954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,4096,1.5813140869140625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3072,1.1917493184407553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2560,0.9773920059204102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,3584,1.3303967793782552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,2048,0.8084597269694009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,256,0.238808536529541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,512,0.30270293553670247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,768,0.3925567944844564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1024,0.47889280319213867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,1536,0.6371018727620442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,128,0.22152214050292968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,64,0.21760320663452148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,65536,32,0.2129151980082194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,8192,0.856271998087565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,10240,1.0682122548421225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,12288,1.268499247233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,16384,1.7043018341064453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,7168,0.7529631932576497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,6144,0.6505941390991211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,5120,0.5501002629597982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,4096,0.44838720957438155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3072,0.3400853474934896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2560,0.289898681640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,3584,0.3955840110778809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,2048,0.23522346814473472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1024,0.1419978618621826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,768,0.11998186906178791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,1536,0.18724160194396972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,512,0.09197013378143311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,256,0.07705279986063639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,128,0.07092800140380859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,64,0.07137920061747233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,32,0.07036266326904297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,10240,0.8299872080485026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,12288,0.9795712153116861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,16384,1.33439572652181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,7168,0.5796554565429688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,8192,0.6670058568318684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,6144,0.5044906616210938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,4096,0.3445354779561361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,5120,0.4201610565185547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3584,0.3058261235555013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,3072,0.2648842652638753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2560,0.22356266975402833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,2048,0.18929600715637207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1024,0.1137610673904419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,1536,0.15171839396158854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,768,0.09782826900482178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,512,0.07502079804738362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,128,0.0577951987584432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,256,0.06378773450851441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,32,0.05813546578089396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,64,0.05819840033849081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,16384,65536,7.120528157552083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,12288,0.8534997304280599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,10240,0.729970105489095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,16384,1.1377386728922525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,7168,0.5161397298177083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,8192,0.5760853449503581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,6144,0.4427029291788737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,12288,65536,5.383168029785156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,4096,0.30332587560017904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,5120,0.3779221216837565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3584,0.270254929860433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2560,0.1988320032755534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,3072,0.23806187311808266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,2048,0.16851520538330078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1536,0.13599573771158854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,1024,0.10191146532694499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,256,0.05756586790084839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,768,0.08779733180999756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,512,0.07125653425852457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,128,0.051351467768351235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,64,0.0519157330195109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,32,0.051429335276285806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,10240,0.6016031901041667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,12288,0.7168170928955078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,16384,0.9511082967122396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,8192,0.48615147272745773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,7168,0.42978881200154617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,6144,0.3728501319885254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,10240,65536,4.57269032796224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,4096,0.2568511962890625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,5120,0.3140629450480143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3584,0.22770026524861656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2048,0.14202879269917806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1536,0.11470080216725667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,3072,0.20021653175354004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,2560,0.17043946584065756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,256,0.04917759895324707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,768,0.07416000366210937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,1024,0.08725653489430746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,512,0.05985173384348551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,64,0.04463253418604533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,32,0.044978133837382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,128,0.04373653332392375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,10240,0.5474442799886068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,12288,0.6506954828898113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,16384,0.8608202616373697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,8192,0.4409813245137532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,7168,0.3892064094543457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,6144,0.3386922518412272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,8192,65536,3.8634005228678383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,4096,0.2341258684794108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,5120,0.2860010782877604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3584,0.20770880381266274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2560,0.15520639419555665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,3072,0.18502933184305828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,2048,0.12895893255869548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1536,0.1044042666753133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,1024,0.08027520179748535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,256,0.0456714669863383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,768,0.068613330523173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,512,0.05545813242594401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,64,0.04173440138498942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,128,0.04144320090611776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,32,0.04145706494649251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,10240,0.4982741355895996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,12288,0.5844042460123698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,16384,0.7742080052693685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,8192,0.40257813135782883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,7168,0.3508597373962402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,6144,0.30599679946899416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,7168,65536,3.4766187032063804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,4096,0.21223786671956382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,5120,0.26105066935221355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3584,0.18781013488769532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2048,0.11797973314921062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,3072,0.16699946721394857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,2560,0.140119473139445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1536,0.09557759761810303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,768,0.063100798924764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,512,0.0508629322052002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,1024,0.07412160237630208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,256,0.04152106841405233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,128,0.03833280007044475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,64,0.038888533910115555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,32,0.03899306853612264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,10240,0.4458922704060872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,12288,0.5249600092569987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,16384,0.6998602549235027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,8192,0.3607605298360189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,7168,0.3169088045756022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,6144,0.2785077412923177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,6144,65536,3.065813446044922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,4096,0.19051307042439777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,5120,0.23445547421773277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3584,0.17139946619669597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,3072,0.15016959508260092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1536,0.08851306438446045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2560,0.12559359868367512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,2048,0.10693973700205486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,1024,0.06892480055491129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,768,0.058178134759267176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,512,0.04732053279876709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,256,0.037835733095804853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,128,0.03412053187688192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,32,0.03370346625645955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,64,0.033529599507649735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,10240,0.38883094787597655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,12288,0.4557557423909505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,16384,0.6060511906941731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,8192,0.31579519907633463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,7168,0.27611627578735354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,6144,0.24269332885742187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,5120,65536,2.737198893229167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,4096,0.16816213925679524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,5120,0.20653440157572428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3584,0.15040213267008465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,3072,0.13138453165690106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2048,0.09491199652353922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,2560,0.11119786898295085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1536,0.07823893229166666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,768,0.051284265518188474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,1024,0.060166398684183754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,512,0.04256853262583415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,256,0.03393599987030029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,128,0.02979946732521057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,64,0.030164267619450885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,32,0.0298250675201416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,10240,0.3716224034627279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,12288,0.4428298632303874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,16384,0.5833333333333333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,8192,0.30165332158406577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,7168,0.26939732233683267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,6144,0.23293439547220865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,4096,65536,2.4150848388671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,4096,0.16383573214213054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,5120,0.1973759969075521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3584,0.14517119725545247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,3072,0.12748053073883056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2560,0.10918613274892171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,2048,0.09135786692301431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1536,0.07556587060292562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,1024,0.057810131708780924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,768,0.049208533763885495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,256,0.032381866375605264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,512,0.040167466799418135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,64,0.029362134138743085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,128,0.02740266720453898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,32,0.029364265998204547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,10240,0.33387734095255533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,12288,0.3981130599975586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,16384,0.5185802777608235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,8192,0.271126397450765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,7168,0.2419637362162272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,6144,0.20927252769470214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,5120,0.17778560320536296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3584,65536,2.2839711507161455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,4096,0.14664427439371747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3584,0.1305450677871704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,3072,0.11318079630533855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2048,0.08432640234629313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,2560,0.09957653681437174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1536,0.0686581293741862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,1024,0.05247893333435059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,512,0.0373525341351827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,768,0.045310934384663895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,256,0.030770132939020794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,128,0.026642133792241413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,32,0.026467200120290118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,64,0.026366933186848955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,10240,0.30788052876790367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,12288,0.364195187886556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,16384,0.47591358820597335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,8192,0.2500714619954427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,7168,0.22110187212626137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,6144,0.18935893376668295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,3072,65536,2.0220096588134764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,5120,0.16333120663960773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,4096,0.13389760653177898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3584,0.11868053277333576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,3072,0.10469226837158203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2560,0.09003413518269857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,2048,0.07604693571726481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1536,0.06083306471506754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,768,0.038771200180053714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,1024,0.04625920057296753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,512,0.03188266754150391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,256,0.026105600595474242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,128,0.022677334149678548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,32,0.022344533602396646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,64,0.022747733195622764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,10240,0.2796330769856771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,12288,0.3310858726501465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,16384,0.4354688008626302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,8192,0.22668693860371908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,7168,0.1989301363627116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,6144,0.17277013460795085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2560,65536,1.851873016357422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,4096,0.12105706532796223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,5120,0.14771626790364584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3584,0.10836053689320882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,3072,0.09490773677825928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2048,0.06905279954274496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,2560,0.08182186285654704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1536,0.05453120072682699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,1024,0.04013226826985677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,512,0.02895466685295105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,768,0.03476053476333618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,128,0.021172267198562623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,256,0.02346773346265157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,64,0.021167999505996703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,32,0.02103040019671122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,10240,0.2532063961029053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,12288,0.30026133855183923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,16384,0.39512745539347327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,8192,0.20526933670043945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,7168,0.18116587003072102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,6144,0.1589471975962321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,2048,65536,1.6923221588134765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,5120,0.13611946105957032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,4096,0.1114741325378418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3072,0.08812692960103354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,3584,0.09892266591389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2048,0.060578131675720216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,2560,0.0748586654663086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1024,0.035810132821400956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,1536,0.04750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,512,0.026181334257125856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,768,0.0297760009765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,128,0.019671465953191122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,256,0.02178986668586731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,32,0.019351466496785482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,64,0.019721599419911702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,10240,0.22460586229960122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,12288,0.26521280606587727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,16384,0.3517535845438639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,8192,0.18346667289733887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,7168,0.16221760114034017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,6144,0.14246932665507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1536,65536,1.5269674936930338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,5120,0.12155946890513103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,4096,0.10018453598022461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3584,0.08926933606465658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,3072,0.07685120105743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2048,0.052393599351247155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,2560,0.06542719999949137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1536,0.040550398826599124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,1024,0.03109440008799235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,128,0.018549333016077675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,512,0.023862399657567344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,256,0.020312533775965372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,768,0.02723306616147359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,64,0.018252799908320107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,32,0.017991467316945394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,10240,0.21228052775065104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,12288,0.2505354722340902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,16384,0.3277258555094401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,8192,0.17346453666687012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,7168,0.15423040390014647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,6144,0.13447893460591634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,1024,65536,1.35600954691569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,4096,0.0946943998336792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,5120,0.11561706860860188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3072,0.07153066794077555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,3584,0.0830677350362142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2560,0.059000531832377114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,2048,0.04702933231989543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1024,0.02932586669921875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,1536,0.03731199900309245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,128,0.01775253415107727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,256,0.019206400712331137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,768,0.02607253392537435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,512,0.022781866788864135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,64,0.017491199572881064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,32,0.017157334089279174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,10240,0.19773866335550944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,12288,0.23561813036600748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,16384,0.3077962557474772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,8192,0.16194987297058105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,7168,0.14482986132303874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,6144,0.12712746461232502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,768,65536,1.2753684997558594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,4096,0.08848640124003092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3584,0.07817280292510986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,3072,0.06581120093663534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,5120,0.10798292954762775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1024,0.02739199995994568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,1536,0.03328426678975423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2048,0.04250880082448323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,2560,0.05372053384780884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,128,0.01698346734046936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,768,0.02441706657409668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,512,0.0214901328086853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,256,0.018554667631785073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,64,0.01694719990094503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,32,0.016472533345222473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,10240,0.1907605330149333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,12288,0.22729066212972007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,16384,0.29677867889404297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,8192,0.15554879506429037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,7168,0.1392309347788493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,512,65536,1.1881440480550132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,6144,0.1215669314066569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3072,0.0625770648320516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,4096,0.08420053323109945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,5120,0.10280213356018067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,3584,0.07371413707733154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1024,0.026209066311518352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,1536,0.03095573385556539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2048,0.03870826562245687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,2560,0.04958933194478353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,768,0.02331733306248983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,256,0.017837866147359212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,512,0.02027413249015808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,128,0.016145066420237223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,64,0.015820800264676412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,32,0.01599360009034475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,10240,0.18804267247517903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,12288,0.22180800437927245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,16384,0.28944212595621743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,8192,0.15351467132568358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,7168,0.13622934023539227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,256,65536,1.1332608540852864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,6144,0.11907413005828857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,5120,0.10159573554992676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3072,0.05996479988098145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,4096,0.08262613614400229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,3584,0.07170026302337647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1024,0.02546986738840739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,1536,0.031205334266026813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2048,0.03726933399836223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,2560,0.047698132197062176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,128,0.016017066438992818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,512,0.020130133628845213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,768,0.02259413401285807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,256,0.017333332697550455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,64,0.015897599856058757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,32,0.015458133816719056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,10240,0.18631572723388673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,12288,0.22070080439249673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,16384,0.2922954559326172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,8192,0.15168746312459308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,7168,0.1342879931131999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,128,65536,1.1165152231852213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,6144,0.11698133150736492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3072,0.05779733260472616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,5120,0.09957546393076579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,4096,0.08058773676554362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,3584,0.06939413547515869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1536,0.03032960096995036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,1024,0.02499413291613261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2560,0.045798401037851974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,2048,0.036634667714436846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,768,0.02251840035120646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,256,0.01692906618118286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,128,0.015871999661127727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,512,0.01988906661669413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,64,0.015655466914176942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,32,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,10240,0.1852373282114665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,12288,0.22040534019470215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,16384,0.28996054331461585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,8192,0.15094399452209473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,7168,0.13393707275390626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,64,65536,1.1166943868001302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,6144,0.11607360045115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3072,0.05623253186543783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,3584,0.06889386971791586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,5120,0.0982805331548055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,4096,0.07988159656524658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1024,0.02509653369585673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2560,0.045109331607818604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,1536,0.03007253408432007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,2048,0.03512959877649943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,768,0.022554665803909302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,512,0.019749333461125694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,128,0.01577173372109731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,256,0.016940800348917644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,64,0.015542399883270264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,32,0.015223466356595359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4096,32,65536,1.113364283243815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,8192,1.5170485178629556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,10240,1.8657056172688802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,12288,2.329956309000651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,7168,1.3038752237955729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,6144,1.154696528116862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,16384,2.9503807067871093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,5120,0.9515071868896484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,4096,0.7732021331787109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2048,0.4096533457438151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3072,0.5999829610188802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1536,0.32520532608032227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,3584,0.6769909540812175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,2560,0.49865172704060867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,256,0.1422101338704427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,128,0.128766934076945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,768,0.2236469268798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,512,0.18159039815266925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,1024,0.26411733627319334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,64,0.12559253374735516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,65536,32,0.12346239884694417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,8192,0.4205151875813802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,10240,0.5204767862955729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,12288,0.6222325642903646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,16384,0.8219008127848306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,7168,0.37013012568155923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,5120,0.26890347798665365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,6144,0.32073278427124025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,4096,0.22183680534362793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2560,0.1513045310974121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3072,0.17306987444559735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,3584,0.1971936066945394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,2048,0.12705386479695638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,768,0.06792746384938558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1536,0.10360639890034992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,1024,0.07928319772084555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,64,0.04488639831542969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,128,0.04420693318049113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,256,0.04815573294957479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,512,0.055172268549601236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,32,0.045176533857981364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,10240,0.4123050689697266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,12288,0.490773328145345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,16384,0.6483434677124024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,7168,0.29412692387898765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,8192,0.33316265741984047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,6144,0.25410453478495276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,4096,0.17724053064982098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,5120,0.2154634634653727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3584,0.15921707153320314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2560,0.12173866430918376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,3072,0.14010987281799317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,2048,0.10233066876729328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1536,0.0839296023050944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,16384,65536,3.331570180257162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,1024,0.06562986771265665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,768,0.05676800012588501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,128,0.03788160085678101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,512,0.046817068258921304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,256,0.04293973445892334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,64,0.03892053365707397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,32,0.03944000005722046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,10240,0.36299839019775393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,12288,0.43204479217529296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,16384,0.5798261642456055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,12288,65536,2.5820266723632814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,8192,0.29452053705851233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,7168,0.25958293279012046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,6144,0.22810346285502114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3072,0.12735466957092284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,4096,0.1574666659037272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,3584,0.14176640510559083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,5120,0.19233919779459635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2560,0.11034239927927654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,2048,0.09181546370188395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1024,0.06065920193990072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,1536,0.07543466885884603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,256,0.03662613232930501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,768,0.052002131938934326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,512,0.04420373439788818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,128,0.033166933059692386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,32,0.034385065237681076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,64,0.033600000540415446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,10240,0.30666774113972983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,12288,0.3663701375325521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,16384,0.4749770800272624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,8192,0.24909440676371256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,7168,0.22043733596801757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,6144,0.18889387448628742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,10240,65536,2.2639284769694012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3584,0.11949973106384278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,5120,0.1621770699818929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,4096,0.1349280039469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,3072,0.10907306671142578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2560,0.09229333400726318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1024,0.05307519833246867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,2048,0.07957546710968018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,1536,0.06587520043055216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,512,0.03928426504135132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,256,0.032307199637095135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,128,0.029522132873535157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,768,0.04512853225072225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,64,0.029428267478942872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,32,0.029958399136861165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,10240,0.27626241048177086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,12288,0.3316682815551758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,16384,0.43181225458780925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,8192,65536,1.8885290781656903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,8192,0.22370880444844565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,7168,0.19995627403259278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,6144,0.16963946024576823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,4096,0.11843199729919433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,5120,0.1441461404164632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3584,0.10756693681081135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,3072,0.09355306625366211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2048,0.0688757340113322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,2560,0.08174826304117838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1024,0.04431573152542114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,1536,0.05664426485697428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,768,0.03845866521199544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,512,0.03338346481323242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,256,0.028926932811737062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,128,0.026903466383616133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,64,0.02717439929644267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,32,0.028474666674931842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,10240,0.25241813659667967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,12288,0.2997034708658854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,16384,0.3945215861002604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,7168,65536,1.7403668721516925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,8192,0.2058549404144287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,7168,0.18235093752543133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,6144,0.1584544022878011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,5120,0.13602879842122395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,4096,0.11217280228932698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3584,0.10146986643473307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,3072,0.09065600236256918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1024,0.04490240017573039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,1536,0.056387201944986975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2560,0.08064640363057454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,2048,0.06767146587371826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,512,0.033478399117787674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,768,0.03946773211161296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,128,0.026791467269261675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,256,0.02882986664772034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,64,0.02709439992904663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,32,0.02595413327217102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,10240,0.22729813257853188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,12288,0.2681941350301107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,16384,0.3566997210184733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,6144,65536,1.5292405446370443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,8192,0.1846197287241618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,6144,0.14109652837117512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,7168,0.16187307039896648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3072,0.07973546981811523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,5120,0.11712426344553631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,4096,0.09836373329162598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,3584,0.09028266270955404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1536,0.049745066960652666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,1024,0.03911786476771037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2560,0.07034773031870524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,2048,0.06010026534398397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,256,0.02446613311767578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,128,0.02276373306910197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,768,0.033920001983642575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,512,0.029262934128443403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,64,0.02244053284327189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,32,0.02327679991722107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,10240,0.19677546819051106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,12288,0.23417065938313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,16384,0.3089674631754557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,5120,65536,1.3633749643961588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,7168,0.1409056027730306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,6144,0.12302079995473225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,8192,0.15922452608744303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,5120,0.10557973384857178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,4096,0.08817173639933268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3584,0.08067519664764404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,3072,0.07040320237477621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2560,0.06133759816487631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,2048,0.0516053318977356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1536,0.04255466858545939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,768,0.029864533742268877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,1024,0.03417706489562988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,128,0.021294933557510377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,512,0.025856000185012818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,256,0.022318933407465616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,64,0.02101759910583496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,32,0.020539732774098714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,10240,0.1964234670003255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,12288,0.22963520685831704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,16384,0.3023765246073405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,4096,65536,1.1775402069091796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,8192,0.15962665875752766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,7168,0.1392661412556966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,6144,0.12205759684244792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,5120,0.10392639636993409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3584,0.0778538703918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,4096,0.08778773148854574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,3072,0.06992959976196289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2560,0.059955199559529625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1536,0.04183679819107056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,2048,0.05025813182195028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,768,0.028782933950424194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,1024,0.033562668164571124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,512,0.02643839915593465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,256,0.022437334060668945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,128,0.020872533321380615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,64,0.020606933037439983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,32,0.020914133389790854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,10240,0.17371840476989747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,12288,0.20640320777893068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,16384,0.2685429255167643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3584,65536,1.1540704091389975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,8192,0.14049386978149414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,7168,0.12388052940368652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,6144,0.10742932955423992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,5120,0.09214826424916586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,4096,0.07845226923624674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3584,0.07091306845347087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,3072,0.062122666835784913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2560,0.05312853256861368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,2048,0.044814932346344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1536,0.0375434676806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,1024,0.030371199051539104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,768,0.026759467522303265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,512,0.02380266586939494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,256,0.020523732900619505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,128,0.019427200158437095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,64,0.019460266828536986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,32,0.01945599913597107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,10240,0.16004692713419597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,12288,0.1871445337931315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,16384,0.24570239384969078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,3072,65536,1.0323978424072267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,8192,0.13066026369730632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,7168,0.11483626365661621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,5120,0.08500373363494873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,6144,0.10032853285471599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3584,0.06388159990310668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,4096,0.07162133057912191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,3072,0.05588266849517822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2560,0.04823360045750936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1536,0.03499946594238281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,2048,0.040720001856486006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,1024,0.02914560039838155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,256,0.020140800873438516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,512,0.022753065824508666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,768,0.025737599531809492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,64,0.01880853374799093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,32,0.018947199980417887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,128,0.018910932540893554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,10240,0.14370667139689128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,12288,0.17179840405782063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2560,65536,0.9349557240804037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,16384,0.22207999229431152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,8192,0.11782186826070148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,7168,0.10630186398824056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,5120,0.07792853514353434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,6144,0.09156266848246256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3584,0.05792853434880575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,4096,0.06388800144195557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,3072,0.04936533371607463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2560,0.04270933469136556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1536,0.031300266583760575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,2048,0.03733546733856201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,1024,0.026025599241256712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,768,0.02378773291905721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,256,0.019160532951354982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,512,0.021154133478800456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,128,0.01798400084177653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,64,0.01784320076306661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,32,0.018004266421000163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,10240,0.13352319399515789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,12288,0.15561386744181316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,16384,0.20247467358907065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,2048,65536,0.8405824025472006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,7168,0.0967146635055542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,8192,0.10958186785380046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,6144,0.08397760391235351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,5120,0.07125973701477051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3584,0.0507861336072286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,3072,0.04428266684214274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2560,0.038942933082580566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,4096,0.05768853425979614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1536,0.028919466336568195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,1024,0.024321067333221435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,768,0.022171733776728313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,2048,0.03384213447570801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,512,0.02028800050417582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,64,0.017246933778127034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,256,0.018142932653427125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,128,0.017110399405161538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,32,0.017258665959040322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,12288,0.13909440040588378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,10240,0.11962986787160237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,16384,0.18066347440083821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1536,65536,0.7668703715006511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,7168,0.08734292984008789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,8192,0.09871893723805746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,5120,0.06264959971110026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,6144,0.0753653367360433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,4096,0.04985386530558268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2560,0.03450773159662883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3072,0.03899413347244263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,3584,0.04426986773808797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,768,0.0209279994169871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1536,0.026374399662017822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,1024,0.022792534033457438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,2048,0.030358399947484332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,512,0.01911679903666178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,256,0.017427200078964235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,128,0.01676266590754191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,64,0.01648533344268799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,32,0.016370133558909098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,12288,0.13695146242777506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,10240,0.1182421366373698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,1024,65536,0.6838794708251953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,16384,0.1726122697194417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,7168,0.08413333098093669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,5120,0.05976853370666504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,8192,0.094540802637736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,6144,0.07258559862772623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3072,0.03705493211746216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,4096,0.04716373284657796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,3584,0.04130133390426636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2560,0.031870933373769124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,768,0.02029333313306173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1536,0.02518613338470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,1024,0.02181653380393982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,2048,0.028344533840815228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,512,0.018645334243774413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,256,0.017131733894348144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,64,0.01598186691602071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,128,0.016109866897265117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,32,0.016433067123095193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,10240,0.1137120008468628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,12288,0.13236160278320314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,16384,0.16885013580322267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,768,65536,0.6541301091512044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,5120,0.05728533267974854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,8192,0.09258879820505778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,7168,0.08147733211517334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,6144,0.06988373597462973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2560,0.030755199988683063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3072,0.03634239832560222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,3584,0.03938453197479248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,4096,0.044609065850575766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,768,0.019769599040349327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1024,0.021309866507848104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,2048,0.02810666759808858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,1536,0.024459733565648397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,128,0.015936000148455302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,256,0.016521599888801575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,512,0.01810879906018575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,64,0.015569067001342774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,32,0.015802666544914246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,12288,0.12465386390686035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,10240,0.10711466471354167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,512,65536,0.6110442479451497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,16384,0.15904320081075032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,8192,0.0861792008082072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,5120,0.05058346589406332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,6144,0.06315946578979492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,7168,0.07542293071746826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2560,0.029156267642974854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3584,0.036771198113759354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,4096,0.03988693157831828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,3072,0.03210560083389282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,2048,0.02622399926185608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1024,0.020428800582885744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,1536,0.023516800006230673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,768,0.01881600022315979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,256,0.01623466710249583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,512,0.017246933778127034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,128,0.015377066532770791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,64,0.01527466674645742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,32,0.015497600038846334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,256,65536,0.5803071975708007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,12288,0.12039146423339844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,10240,0.10352426369984943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,16384,0.1565930684407552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,7168,0.07302719751993815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,5120,0.04854933420817058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,8192,0.08289706707000732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,6144,0.06110719839731852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3072,0.03149333397547404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,3584,0.033829331398010254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,4096,0.038441598415374756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2560,0.028589866558710736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1024,0.020487467447916664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,1536,0.023127466440200806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,768,0.018708266814549766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,2048,0.02566933234532674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,512,0.017412267128626504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,64,0.015246933698654175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,256,0.015920000274976094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,128,0.015475199619928996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,32,0.015065600474675497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,128,65536,0.5726773579915364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,12288,0.12008533477783204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,10240,0.1019701321919759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,16384,0.15438186327616374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,6144,0.06000853379567465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,8192,0.08326506614685059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,5120,0.04673813184102376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,7168,0.07299839655558268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3072,0.031142399708429976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,3584,0.034092799822489424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,4096,0.03775360186894734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2560,0.02820906639099121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,2048,0.02592960000038147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1536,0.02295359969139099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,1024,0.019886932770411172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,768,0.018473599354426065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,256,0.015819733341534935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,512,0.017257599035898845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,128,0.015278933445612588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,64,0.015007999539375306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,32,0.015096533298492431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,64,65536,0.5678410847981771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,10240,0.1017568031946818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,12288,0.11936426957448323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,16384,0.15286506017049153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,8192,0.08232106367746989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,7168,0.07169280052185059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,6144,0.05939626693725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,5120,0.04673706690470378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2560,0.028483200073242187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3072,0.030921600262324017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,4096,0.036424533526102705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,3584,0.03405653238296509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,2048,0.02540480097134908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1024,0.020104533433914183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,768,0.01851093371709188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,1536,0.022938666741053264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,256,0.01601599951585134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,512,0.017139200369517008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,128,0.015362133582433065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,64,0.01502293348312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,32,0.015310933192571005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2048,32,65536,0.5627295811971028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,8192,0.7681109110514324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,7168,0.6784447987874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,10240,0.9524170557657877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,12288,1.1566752115885417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,4096,0.40685440699259445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,6144,0.5844160079956054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,16384,1.58907839457194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,5120,0.494813855489095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3584,0.36330560048421223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1024,0.13538133303324382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,1536,0.17774826685587566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2560,0.2631850719451904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,2048,0.22707734107971192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,3072,0.31254825592041013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,256,0.07726293404897054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,768,0.12239573001861573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,64,0.07260053157806397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,128,0.07160106499989828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,512,0.1010474681854248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,65536,32,0.07189866701761881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,8192,0.21248106956481932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,10240,0.262829860051473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,12288,0.31645545959472654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,16384,0.41277332305908204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,6144,0.16767573356628418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,7168,0.18905599912007648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,5120,0.1452309290568034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,4096,0.11991893450419108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3072,0.09558613300323486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2560,0.08443946838378906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,3584,0.10851733684539795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,2048,0.07127253214518228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,768,0.04288533528645833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1536,0.05984106858571371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,1024,0.04876266717910767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,512,0.03715200026830037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,256,0.03186773260434468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,128,0.029548799991607665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,64,0.029945600032806396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,32,0.0294869323571523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,10240,0.20813973744710287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,12288,0.25099093119303384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,16384,0.32644265492757163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,8192,0.1703722635904948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,7168,0.1533919970194499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,6144,0.13404800097147623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,4096,0.09840853214263916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3584,0.0883850653966268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,5120,0.11637866497039795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,16384,65536,1.6318698883056642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2560,0.06918613115946451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,2048,0.06010666688283285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,3072,0.0794058640797933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1536,0.05192000071207682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,1024,0.041458133856455484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,768,0.036950401465098065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,512,0.03219306667645772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,128,0.02645333409309387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,256,0.028038400411605834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,64,0.02654613256454468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,32,0.025455999374389648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,10240,0.18408320744832357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,12288,0.21802345911661783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,16384,0.28912213643391926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,12288,65536,1.2767903645833334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,8192,0.1511306603749593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,7168,0.13385920524597167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,6144,0.11817066669464112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,5120,0.10511360168457032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,4096,0.08590826988220215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3584,0.07705279986063639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,3072,0.0694645325342814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2560,0.06212586561838786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1024,0.0359605352083842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,2048,0.0531274676322937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,1536,0.04429440100987752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,768,0.0321834663550059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,512,0.02803093393643697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,256,0.025126399596532185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,128,0.02305493354797363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,64,0.022436267137527464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,32,0.022427733739217123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,10240,0.1531178633371989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,12288,0.18436800638834636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,16384,0.24024747212727865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,10240,65536,1.1182229359944662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,8192,0.1249951998392741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,7168,0.11368746757507324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,6144,0.09827520052591959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3584,0.06793813705444336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,5120,0.0862506628036499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,3072,0.05938453276952108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,4096,0.07352213064829508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1536,0.03755093415578206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2560,0.052653865019480384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,2048,0.04463573296864827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,1024,0.03137279947598775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,768,0.028014934062957762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,128,0.02023893396059672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,256,0.021333332856496176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,512,0.0253440002600352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,64,0.020795732736587524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,32,0.020871466398239134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,10240,0.15133439699808757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,12288,0.1782698631286621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,8192,65536,0.943287467956543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,16384,0.23808746337890624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,5120,0.08680213292439778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,8192,0.12406613032023113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,6144,0.1005738655726115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,7168,0.11039040088653565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,4096,0.07278933525085449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3584,0.06512426535288493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2560,0.051062401135762533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,3072,0.05915946563084921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1536,0.03696533441543579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,2048,0.04410560131072998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,768,0.028051199515660603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,1024,0.03099199930826823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,512,0.0241482675075531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,256,0.021918932596842446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,128,0.02073600093523661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,64,0.020848000049591066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,32,0.020707199970881142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,10240,0.1297045310338338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,12288,0.1538090705871582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,7168,65536,0.8966794967651367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,16384,0.2080842653910319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,5120,0.07547626495361329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,6144,0.08418560028076172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,7168,0.09530239899953206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,8192,0.10642986297607422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,4096,0.0639573335647583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3584,0.056804267565409336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2560,0.04496959845225017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,3072,0.051040001710255946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,2048,0.03910719950993856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1536,0.03270186583201091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,1024,0.027803732951482134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,768,0.025384533405303954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,512,0.02290453314781189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,256,0.02023680011431376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,128,0.019006933768590292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,64,0.019511467218399046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,32,0.01936533252398173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,10240,0.1162549336751302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,12288,0.1355402628580729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,6144,65536,0.7885322570800781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,16384,0.18103466033935547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,5120,0.06671786308288574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,6144,0.0772607962290446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,7168,0.08540373643239339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,8192,0.09662506580352784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,4096,0.05649280150731405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3584,0.051867731412251795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,3072,0.04597119887669881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2560,0.04107093413670858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,2048,0.03543999989827474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1536,0.03099626700083415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,1024,0.026162133614222212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,768,0.024472532669703166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,512,0.021117866039276123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,256,0.019910399119059244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,128,0.018732800086339315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,64,0.019063466787338258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,32,0.018895999590555827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,12288,0.12018880049387615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,5120,65536,0.6790378570556641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,10240,0.09988906383514404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,16384,0.15661973953247071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,6144,0.06780906518300375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,7168,0.07621653079986572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,5120,0.05892906586329142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,8192,0.08543573220570883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3072,0.039628799756368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,4096,0.04931413332621257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,3584,0.04404053290685018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2560,0.0357258677482605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,768,0.021724800268809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1024,0.023038933674494423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,1536,0.02755413254102071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,2048,0.0315338671207428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,256,0.018745599190394084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,512,0.020374399423599244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,128,0.017760000626246133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,64,0.017975467443466186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,32,0.01784106691678365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,4096,65536,0.5919594446818034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,12288,0.11519359747568766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,10240,0.09790186882019043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,16384,0.15425492922465006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,8192,0.0831775983174642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,6144,0.0663861314455668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,5120,0.058141867319742836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,7168,0.07438399791717529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,4096,0.046215466658274335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3072,0.037878398100535074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,3584,0.040889600912729904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2560,0.03333013256390889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,2048,0.029397332668304445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1536,0.02558186650276184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,1024,0.021986132860183714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,768,0.02110613385836283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,256,0.018116267522176106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,512,0.018968532482782997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,128,0.017427200078964235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,64,0.017661867539087932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,32,0.017848533391952515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3584,65536,0.5811594645182292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,10240,0.08874453703562418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,12288,0.10389653046925862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,16384,0.13811945915222168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,8192,0.07602240244547526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,7168,0.06809066931406657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,5120,0.05213119983673096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,6144,0.060013866424560545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,4096,0.04382719993591309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3584,0.039139199256896975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,3072,0.03554986715316773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2560,0.032110933462778726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1024,0.02195733388264974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,2048,0.028917332490285236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,1536,0.0260917325814565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,768,0.020338133970896403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,512,0.019194666544596353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,128,0.01690666675567627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,256,0.01776533325513204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,64,0.017170133193333943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,32,0.01723519961039225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,3072,65536,0.5179818789164226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,12288,0.0973855972290039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,10240,0.08386452992757162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,16384,0.12921813329060872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,8192,0.0714186668395996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,7168,0.0641205350557963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,6144,0.05689493417739868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,5120,0.04904426733652751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,4096,0.040591998895009355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3584,0.03718400001525879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,3072,0.034456535180409746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2560,0.031251200040181476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,2048,0.028257066011428834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1024,0.022039467096328737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,1536,0.024378667275110878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,512,0.01926079988479614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,768,0.02032533288002014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,256,0.017758933703104655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,128,0.01704853375752767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,64,0.017086933056513466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,32,0.017164800564448038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2560,65536,0.4952746709187825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,10240,0.07713279724121094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,16384,0.11559360027313233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,12288,0.08930559953053793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,8192,0.06443626483281453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,5120,0.04304746786753337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,6144,0.05044053395589193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,7168,0.05804800192515055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,4096,0.03658879995346069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2560,0.028526933987935384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3072,0.03070080081621806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,3584,0.033828266461690265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,2048,0.025586134195327757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,512,0.017901867628097534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1024,0.02010986606280009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,1536,0.022315732638041177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,768,0.019080533583958944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,256,0.017114667097727458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,128,0.01653333306312561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,32,0.016484266519546507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,64,0.016359466314315795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,2048,65536,0.43000106811523436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,10240,0.0735157330830892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,12288,0.08520212968190512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,8192,0.05798506736755371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,16384,0.10586559772491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,7168,0.0512661337852478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,5120,0.03849386771519979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,6144,0.044801068305969236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,4096,0.03307626644770305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3072,0.028943999608357744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,3584,0.03094080090522766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2048,0.023716266949971518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,2560,0.02600319981575012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1536,0.021350399653116862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,768,0.018155733744303383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,512,0.0173962672551473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,1024,0.019385600090026857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,256,0.016587733229001363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,64,0.01602133313814799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,128,0.01583573321501414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,32,0.016114133596420287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1536,65536,0.4041023890177409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,10240,0.06961706479390463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,8192,0.05322879950205485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,12288,0.08143680095672608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,16384,0.10405440330505371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,6144,0.04211626847585042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,5120,0.03643413384755452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,7168,0.04744213422139486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,4096,0.03161279956499736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2048,0.022678399085998537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3584,0.029399466514587403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,2560,0.024204800526301064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,3072,0.02646399935086568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,768,0.017653334140777587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1024,0.018771199385325114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,1536,0.02082560062408447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,512,0.01692053278287252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,256,0.01613759994506836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,128,0.015664000312487283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,64,0.015660799543062844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,32,0.015507200360298156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,1024,65536,0.35693546930948894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,8192,0.05043840010960897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,10240,0.06529173453648886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,12288,0.07839039961496988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,16384,0.09994346300760905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,7168,0.044494934876759845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,4096,0.029237333933512372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,6144,0.0399829347928365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,5120,0.035336534182230636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3584,0.027381332715352376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2560,0.023772799968719484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,3072,0.025900799036026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,2048,0.02220053275426229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1536,0.020053333044052123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,1024,0.017992534240086875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,768,0.01722773313522339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,512,0.0165994664033254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,256,0.015889066457748412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,128,0.015301332871119181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,64,0.015421866377194723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,65536,0.33828372955322267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,768,32,0.015527466932932535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,12288,0.07290773391723633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,8192,0.04681493441263835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,10240,0.06053760051727295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,16384,0.09590186278025309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,5120,0.03274986743927002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,4096,0.028253867228825884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,7168,0.04138666788736979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,6144,0.037356801827748615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2560,0.022986666361490885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3072,0.024563199281692503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,2048,0.021342933177947998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,3584,0.026748800277709962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1536,0.01953279972076416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,1024,0.017628800868988038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,512,0.01623679995536804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,768,0.017091200749079386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,256,0.0157642662525177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,65536,0.31744321187337243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,64,0.01504853367805481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,128,0.015160533785820007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,512,32,0.015185067057609558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,8192,0.0439850648244222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,10240,0.05628373225529989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,12288,0.06907626787821451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,16384,0.08977813720703125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,5120,0.03174399932225545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,7168,0.03914346694946289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,4096,0.02794559995333354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,6144,0.03413653373718262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2560,0.02290560007095337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3584,0.026261333624521894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,3072,0.024357332785924276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,2048,0.021361066897710165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1536,0.019372800985972084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,1024,0.017853866020838417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,768,0.017013333241144814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,512,0.016484266519546507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,65536,0.30219306945800783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,256,0.015490133563677469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,128,0.015020799636840821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,64,0.01502293348312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,256,32,0.015278933445612588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,8192,0.04208853244781494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,10240,0.051812267303466795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,12288,0.0627125342686971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,16384,0.08544747034708658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,7168,0.03837333520253499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,5120,0.030161066850026445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,6144,0.03365653355916341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,4096,0.027064534028371175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2560,0.022411733865737915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,2048,0.020678400993347168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3584,0.025476266940434773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,3072,0.0238154669602712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1024,0.017525333166122436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,1536,0.019113600254058838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,768,0.016801067193349204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,512,0.016152532895406087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,256,0.01548906664053599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,128,0.01511253317197164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,65536,0.2956970532735189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,64,0.015005866686503092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,128,32,0.015027200182278952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,8192,0.040085331598917646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,12288,0.06215466658274332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,10240,0.050629333655039466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,16384,0.08250133196512857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,6144,0.03342506488164266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,4096,0.026903466383616133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,5120,0.029985066254933673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,7168,0.0361631989479065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3584,0.02534079949061076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,3072,0.023652267456054688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2048,0.020742400487263998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,2560,0.022027732928593953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1024,0.017525333166122436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,1536,0.019010132551193236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,768,0.016582399606704712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,512,0.015922133127848306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,65536,0.2930090586344401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,128,0.015059199929237366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,256,0.015294933319091797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,32,0.01497706671555837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,64,64,0.014869333306948344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,8192,0.0384607990582784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,12288,0.05927466551462809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,10240,0.048944000403086343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,16384,0.08278720378875733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,4096,0.02677653431892395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,5120,0.02988053361574809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,6144,0.03306666612625122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,7168,0.03611413240432739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3072,0.02381226619084676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,3584,0.025337600708007814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2560,0.02210986614227295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,2048,0.020665599902470907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1536,0.019050665696461997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,1024,0.017347200711568197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,768,0.016536532839139303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,512,0.015801599621772765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,65536,0.29175999959309895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,256,0.015438933173815408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,128,0.014905599753061929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,64,0.014769066373507181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1024,32,32,0.014824533462524414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,7168,0.5193536122639973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,8192,0.5779712041219075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,10240,0.7155786514282226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,12288,0.8503146489461264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,6144,0.45018774668375655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,5120,0.37144425710042317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,4096,0.3040661176045736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,16384,1.1186965942382812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3584,0.2683039983113607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2560,0.20237332979838052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,2048,0.16906026204427083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,3072,0.23964373270670572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1536,0.13754132588704426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,1024,0.1026911973953247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,256,0.06371946732203165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,512,0.07243306636810302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,768,0.08833386898040771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,128,0.058710400263468424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,32,0.05960213343302408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,65536,64,0.059723734855651855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,8192,0.16494933764139813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,10240,0.2035381317138672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,12288,0.2349109331766764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,16384,0.3124917348225912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,7168,0.14684160550435382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,5120,0.11122773488362629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,6144,0.1315029303232829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,4096,0.09445546468098959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2560,0.06739199956258138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3072,0.07601173718770346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,3584,0.08505067030588785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,2048,0.05785599946975708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1536,0.04940906763076782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,1024,0.0407968004544576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,768,0.03690346479415894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,512,0.03205546736717224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,128,0.027438932657241823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,256,0.02773226698239644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,64,0.027246934175491334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,32,0.026744532585144042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,10240,0.15835199356079102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,12288,0.18579840660095215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,16384,0.2475850741068522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,8192,0.1301024039586385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,7168,0.11570453643798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,6144,0.10342826843261718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,5120,0.0885642687479655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,4096,0.07501227060953776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3584,0.06845973332722982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,16384,65536,1.2375253041585288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2560,0.05429759820302328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,2048,0.047038932641347245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,3072,0.061093334356943765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1536,0.041127467155456544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,768,0.029441066582997638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,1024,0.03272746602694194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,512,0.026230400800704955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,256,0.023425066471099855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,128,0.0209824005762736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,32,0.02153173287709554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,64,0.02179093360900879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,10240,0.14180053075154622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,12288,0.16636692682902018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,16384,0.22070080439249673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,12288,65536,0.9549311955769857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,8192,0.11670719782511393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,7168,0.10415039857228596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,5120,0.0819434642791748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,6144,0.09322026570638021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3072,0.05601386626561483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2560,0.04967679977416992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,4096,0.06821226278940837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,3584,0.062410668532053626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1536,0.03667200009028117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,2048,0.04290453195571899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,1024,0.03052373329798381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,768,0.027669332424799603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,512,0.0249834676583608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,256,0.02436586618423462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,128,0.02259626587231954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,64,0.022821333010991415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,32,0.0232149342695872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,10240,0.12120640277862549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,12288,0.14029653867085773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,16384,0.18461546897888184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,10240,65536,0.8591530481974283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,8192,0.10038399696350098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,7168,0.08922026952107748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,6144,0.07991147041320801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,5120,0.07189119656880696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3072,0.04790293375651042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,3584,0.05347520112991333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,4096,0.059463465213775636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2560,0.04295146862665812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1024,0.027088000377019243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,2048,0.036957867940266925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,768,0.02537279923756917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,1536,0.03189226587613424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,128,0.019282132387161255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,512,0.022833067178726196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,64,0.019411200284957887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,256,0.020121600230534872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,32,0.019537067413330077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,12288,0.12818773587544757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,8192,65536,0.7170581181844076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,10240,0.11133333047231038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,16384,0.16787199974060057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,6144,0.07404692967732748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,5120,0.06383573214213054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,8192,0.09154773553212484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,7168,0.08210773468017578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,4096,0.05464959939320883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3072,0.044361599286397296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,3584,0.049388798077901204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2560,0.03910719950993856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,768,0.023413334290186563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1024,0.024915200471878052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,1536,0.029412267605463664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,2048,0.03464853366216024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,512,0.021741867065429688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,64,0.01939093271891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,256,0.020246400435765585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,128,0.019275732835133872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,32,0.019528534015019736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,10240,0.0983136018117269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,12288,0.11442773342132569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,7168,65536,0.6609888076782227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,16384,0.14994133313496907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,5120,0.05791786511739096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,6144,0.06644479831059774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,8192,0.08213439782460531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,7168,0.07389439741770426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,4096,0.04972586631774902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3072,0.04052053292592366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,3584,0.045102934042612716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2560,0.03599040110905965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,2048,0.03195093274116516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1024,0.024447999397913613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,1536,0.02806719938913981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,768,0.02290133237838745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,512,0.0206112007300059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,256,0.019003732999165853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,128,0.018252799908320107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,64,0.018565332889556883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,32,0.018522665898005167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,6144,65536,0.5768576304117838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,12288,0.10432533423105876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,16384,0.137007999420166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,10240,0.09021333058675131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,6144,0.0615775982538859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,8192,0.07642880280812582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,5120,0.05384000142415365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,7168,0.0684554656346639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,4096,0.04587626854578654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3072,0.037373868624369304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,3584,0.042130132516225174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2560,0.03362880150477092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,768,0.021220266819000244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1536,0.026537599166234334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,2048,0.029997867345809937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,1024,0.023430399099985757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,512,0.020023467143376668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,256,0.0182805339495341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,128,0.017598932981491087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,64,0.01767680048942566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,32,0.01770346760749817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,5120,65536,0.5275466601053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,12288,0.0921781301498413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,10240,0.07968213558197021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,16384,0.11980480353037518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,8192,0.06841813723246257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,7168,0.06146666606267294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,5120,0.047127465407053634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,6144,0.05470826625823975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3584,0.03622506856918335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,4096,0.03978026707967122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2560,0.030194133520126343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,3072,0.03338133494059245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,2048,0.027464532852172853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1536,0.024426666895548503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,1024,0.021014400323232017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,768,0.020195200045903524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,512,0.018873600165049235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,128,0.017117865880330405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,256,0.017566933234532674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,64,0.017193599541982015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,32,0.017141334215799966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,4096,65536,0.4529578526814778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,12288,0.08686933517456055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,10240,0.0759488026301066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,16384,0.11249173482259114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,8192,0.06394240061442057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,7168,0.058024533589680995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,6144,0.0525056004524231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,5120,0.04561173518498739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,4096,0.0383349339167277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3584,0.034946131706237796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,3072,0.03273599942525228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2560,0.029950932661692305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,2048,0.02683519919713338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1536,0.024204800526301064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,1024,0.021228800217310585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,768,0.02036906679471334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,512,0.019064533710479736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,128,0.017656532923380534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,256,0.01800533334414164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,64,0.01755946675936381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,32,0.017691733439763387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3584,65536,0.4227754592895508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,10240,0.0731498638788859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,16384,0.10514026482899982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,8192,0.061300265789031985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,12288,0.08282559712727865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,7168,0.055087999502817786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,6144,0.050220799446105954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,4096,0.03654613494873047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,5120,0.04239893356959025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3584,0.033292800188064575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,3072,0.030499200026194256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2048,0.02570880055427551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,2560,0.028066132465998334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1536,0.023902932802836098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,1024,0.019912532965342202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,512,0.0180842657883962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,768,0.019062399864196777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,256,0.016989866892496742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,128,0.016598400473594666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,64,0.01665066679318746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,32,0.016693333784739174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,3072,65536,0.4079786618550618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,12288,0.07646826903025308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,10240,0.06662720044453939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,8192,0.057062399387359616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,16384,0.09769279956817627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,7168,0.051130668322245276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,6144,0.0448415994644165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,4096,0.03360746701558431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,5120,0.03875733216603597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3584,0.03101759950319926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2560,0.02658560077349345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,2048,0.024196267127990723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,3072,0.028911999861399335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,768,0.018894932667414346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1536,0.021967999140421548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,1024,0.01971093416213989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,512,0.017935999234517417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,256,0.016876800855000814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,128,0.01649066706498464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,64,0.016727467377980552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,32,0.01649493376413981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2560,65536,0.3638506571451823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,12288,0.08114559650421142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,10240,0.07035733064015706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,8192,0.05421973466873169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,16384,0.10367360115051269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,5120,0.03664000034332275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,4096,0.032126933336257935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,7168,0.04816746711730957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,6144,0.04246400197347005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3584,0.029638399680455525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,3072,0.02758079965909322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2560,0.02521066665649414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,2048,0.022366933027903237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1024,0.018385066588719686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,1536,0.02055466572443644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,768,0.01798506577809652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,512,0.01730666756629944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,256,0.016210132837295534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,64,0.015960533420244852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,128,0.015853866934776306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,32,0.016008533040682473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,2048,65536,0.34236052831013997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,8192,0.04708693424860637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,12288,0.06607360045115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,10240,0.05707519849141439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,16384,0.08379519780476888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,4096,0.029275733232498168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,6144,0.03688640197118123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,7168,0.04103786547978719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,5120,0.032817065715789795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3584,0.027075199286142985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,3072,0.025487999121348064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2560,0.023614933093388878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,2048,0.021593600511550903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1536,0.020670932531356812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,1024,0.018620800971984864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,768,0.0180351992448171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,512,0.016911999384562174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,256,0.015979733069737753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,65536,0.31222613652547204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,64,0.016132266322771708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,128,0.015964800119400026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1536,32,0.015887999534606935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,10240,0.054738132158915195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,8192,0.04240959882736206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,12288,0.07017707029978434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,16384,0.09024106661478679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,6144,0.03521066506703695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,4096,0.02774080038070679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,5120,0.031164799133936567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,7168,0.03894720077514648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3072,0.024621866146723428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,3584,0.025278933842976886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2560,0.022459733486175536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,2048,0.020827732483545938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1536,0.019266132513682047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,768,0.01723946730295817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,1024,0.0176256000995636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,512,0.016545066237449647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,65536,0.27557973861694335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,256,0.015708800156911215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,128,0.015508266290028891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,32,0.015367466211318969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,1024,64,0.015577600399653117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,10240,0.04809600114822388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,8192,0.03938026825586955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,12288,0.05799039999643961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,16384,0.07668266296386719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,7168,0.03572266499201457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,4096,0.02831466595331828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,5120,0.03195733428001404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,6144,0.033061333497365314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2560,0.022716800371805825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3584,0.02636373241742452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,2048,0.02132373253504435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,3072,0.025153066714604693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1536,0.019307732582092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,1024,0.017783466974894205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,512,0.016380799810091655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,768,0.017032533884048462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,65536,0.2599167982737223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,256,0.015631999572118124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,64,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,128,0.015275733669598899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,768,32,0.015273599823315939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,10240,0.04453866481781006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,12288,0.05345066785812378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,8192,0.03727999925613403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,16384,0.07306986649831136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,4096,0.025320533911387128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,6144,0.03127893408139547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,7168,0.03410773277282715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,5120,0.02871893246968587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3584,0.02395306626955668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,3072,0.022679466009140014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2560,0.021451733509699502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,2048,0.01990933418273926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1024,0.016884267330169678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,1536,0.01846933364868164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,65536,0.2473994731903076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,768,0.01671253244082133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,512,0.016107733050982155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,256,0.015385599931081137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,128,0.015211733182271323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,64,0.015362133582433065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,512,32,0.01527253290017446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,8192,0.03552746772766113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,10240,0.042022399107615155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,12288,0.04981120030085246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,16384,0.06819413503011068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,4096,0.024626133839289348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,5120,0.027315199375152588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,6144,0.03095253308614095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,7168,0.03278613289197286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3072,0.022502400477727256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2048,0.01954666574796041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,2560,0.020832000176111858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,3584,0.023307732741038003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,65536,0.2342090606689453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1536,0.018351999918619792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,1024,0.017178666591644288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,768,0.016800000270207723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,512,0.01586133340994517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,256,0.015408000349998474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,128,0.015117866794268289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,32,0.015014400084813436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,256,64,0.014862933754920959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,16384,0.0636298656463623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,8192,0.03470720052719116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,12288,0.04615039825439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,10240,0.0403168002764384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,4096,0.02459733287493388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,5120,0.02709760069847107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,7168,0.032732800642649336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,6144,0.029227733612060547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3584,0.023104000091552734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,3072,0.022106667359670006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2560,0.020696532726287842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,2048,0.019429334004720054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,65536,0.22498879432678223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1536,0.01800640026728312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,1024,0.016819200913111367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,512,0.015629866719245912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,768,0.01655893325805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,256,0.015209600329399109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,128,0.014964266618092855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,64,0.015103999773661295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,128,32,0.014869333306948344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,8192,0.03354666630427043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,10240,0.0384768009185791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,16384,0.061610666910807285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,12288,0.044766934712727864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,4096,0.024510933955510458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,5120,0.026840533812840777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,6144,0.029708800713221233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,7168,0.032009599606196086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2560,0.020567466815312706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3584,0.02330026626586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,3072,0.022066134214401244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,2048,0.01939199964205424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,65536,0.2247584025065104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1536,0.01787733236948649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,512,0.01583146651585897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,1024,0.017011199394861856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,768,0.016184533635775246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,256,0.015383467078208923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,128,0.015080533425013223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,64,0.015026133259137472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,64,32,0.014923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,12288,0.043133866786956784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,10240,0.037699198722839354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,8192,0.03296213348706563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,16384,0.059704534212748205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,4096,0.024234666426976522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,5120,0.02701759934425354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,7168,0.03179733355840047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,6144,0.029524266719818115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3584,0.0231605331103007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,3072,0.021962666511535646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2560,0.020520534118016562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,2048,0.019223467508951823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,65536,0.22130239804585777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1536,0.017810134092966716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,1024,0.01673813263575236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,768,0.01636799971262614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,512,0.015691733360290526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,256,0.01495573321978251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,128,0.01498133341471354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,64,0.014957867066065469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,768,32,32,0.014803199966748556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,7168,0.340936533610026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,8192,0.38181333541870116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,10240,0.4725141207377116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,12288,0.5766239802042643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,5120,0.2503135999043783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,6144,0.29654080073038735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,4096,0.20578239758809408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,16384,0.7490389506022136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3584,0.18731519381205242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2560,0.13995520273844403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,3072,0.16203413009643555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,2048,0.11875840028127034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1024,0.07669119834899903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,1536,0.10346773465474446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,256,0.04998826583226522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,512,0.0577023983001709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,64,0.04526400168736776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,128,0.044652799765268966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,768,0.06632106701533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,65536,32,0.04743893146514892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,8192,0.11457173029581706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,10240,0.13877654075622559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,12288,0.16426560084025066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,16384,0.21312853495279946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,6144,0.09067306518554688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,7168,0.10255359808603923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,5120,0.07926826477050782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,4096,0.06690986951192221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3584,0.06053653160730997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,3072,0.05440213282903036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2560,0.04853440125783284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,2048,0.04267093340555827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1536,0.036414933204650876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,1024,0.03069546620051066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,768,0.027988266944885255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,512,0.025498666365941364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,128,0.020513067642847695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,256,0.022190932432810465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,64,0.020692267020543418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,32,0.02081706722577413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,10240,0.11654613018035889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,12288,0.13636587460835775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,16384,0.17333440780639647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,7168,0.08644586404164632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,8192,0.09619946479797363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,6144,0.0755295991897583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,5120,0.06600746711095175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3584,0.051151998837788905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,4096,0.05709226528803507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,16384,65536,0.8317994435628255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2560,0.04101333220799764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,2048,0.036917332808176676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1536,0.03264106710751851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,3072,0.04629546801249186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,1024,0.02672106623649597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,768,0.025150932868321735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,512,0.022946133216222128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,256,0.02007466753323873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,128,0.0190720001856486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,64,0.019461333751678467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,32,0.01962453325589498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,10240,0.10212586720784504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,12288,0.11982186635335286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,12288,65536,0.6734517415364583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,16384,0.14982399940490723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,8192,0.0849951982498169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,5120,0.059411199887593594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,7168,0.07711466948191324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,6144,0.07421759764353433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3072,0.043091201782226564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,4096,0.05116159915924072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2560,0.03894613186518352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,3584,0.047437866528828934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1536,0.029358933369318645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,1024,0.025497599442799883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,2048,0.03404266834259033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,768,0.023577600717544556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,512,0.021821866432825722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,256,0.01941546599070231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,128,0.018685867389043175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,64,0.018913066387176512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,32,0.018908800681432088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,12288,0.09828906853993734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,10240,0.08551999727884928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,10240,65536,0.5790229161580404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,16384,0.12703146934509277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,8192,0.07084373633066812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,5120,0.050931199391682946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,7168,0.06423679987589517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,6144,0.060184534390767416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3584,0.04029866854349772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,3072,0.03659520149230957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,4096,0.04359253247578938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2560,0.032657066980997726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,2048,0.02958400050799052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1536,0.026114133993784587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,1024,0.023118933041890465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,768,0.021625600258509316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,256,0.01835626761118571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,512,0.019662932554880778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,128,0.017795199155807497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,64,0.017882666985193887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,32,0.017939200003941856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,8192,65536,0.4777077356974284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,10240,0.0824394702911377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,12288,0.09622933069864908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,16384,0.12053226629892985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,8192,0.06941119829813638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,5120,0.04916160106658936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,7168,0.0629258672396342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,6144,0.05645440022150675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,4096,0.040616532166798905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3584,0.03722026745478312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,3072,0.03426346778869629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2560,0.030664533376693726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1536,0.024175999561945598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,2048,0.027163734038670857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,1024,0.02143999934196472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,768,0.019589332739512126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,512,0.018870399395624796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,256,0.01776319940884908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,128,0.01758400003115336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,64,0.017534933487574258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,32,0.017973333597183228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,7168,65536,0.458189868927002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,10240,0.07219413121541342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,12288,0.08513813018798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,16384,0.10679787000020344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,8192,0.06186240116755167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,5120,0.04404266675313313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,6144,0.05028160015741984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,7168,0.05522240002950033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3584,0.034968535105387374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,4096,0.03881813287734985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,3072,0.03206400076548259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2560,0.029007999102274577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,2048,0.026485333840052288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1024,0.021613866090774536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,1536,0.024332799514134727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,768,0.01939520041147868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,512,0.01880533297856649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,256,0.01725119948387146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,128,0.017144532998402913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,64,0.01705066760381063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,32,0.017174400885899863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,6144,65536,0.4002271970113118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,12288,0.07696639696756999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,10240,0.06709760030110677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,16384,0.09838079611460368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,8192,0.05767999887466431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,6144,0.04683306614557902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,5120,0.04162559906641643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,7168,0.05187840064366659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3584,0.033156265815099076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,4096,0.03634239832560222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2560,0.027859199047088622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,3072,0.030436267455418903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,2048,0.02568639914194743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1536,0.02379306753476461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,1024,0.021197867393493653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,768,0.019766400257746376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,512,0.018849066893259683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,256,0.017924267053604125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,128,0.01705066760381063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,64,0.017032533884048462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,32,0.017166932423909508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,5120,65536,0.3768416086832682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,10240,0.05962773164113363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,16384,0.08500586350758871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,12288,0.06900479793548583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,8192,0.050153601169586184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,7168,0.04616106748580932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,4096,0.031231999397277832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,6144,0.041275731722513836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,5120,0.036236798763275145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3584,0.02922133406003316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,3072,0.026950399080912273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2560,0.025179733832677204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,2048,0.023476266860961915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1024,0.01919893423716227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,1536,0.021581866343816123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,768,0.01821440060933431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,512,0.01771199901898702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,256,0.016505600015322367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,128,0.01632426679134369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,64,0.016477866967519125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,32,0.01673813263575236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,4096,65536,0.3142005284627279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,8192,0.04728000164031983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,12288,0.06286293268203735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,10240,0.05765546560287475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,16384,0.07867519855499268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,7168,0.04276906649271647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,6144,0.03789546489715576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,4096,0.03033706744511922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,5120,0.03382506767908732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3584,0.027987200021743774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,3072,0.02586666742960612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2560,0.024117332696914674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,2048,0.022706133127212525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1536,0.021678932507832847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,1024,0.018222934007644652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,768,0.018221867084503175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,512,0.017541333039601644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,256,0.01649066706498464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,128,0.01605226695537567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,64,0.01609493295351664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,32,0.016302933295567833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3584,65536,0.2892064094543457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,10240,0.05741440057754517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,8192,0.04419306516647339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,12288,0.06151893138885498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,16384,0.07428586483001709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,7168,0.04053013324737549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,6144,0.03529599905014038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,5120,0.03186239997545878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,4096,0.02813653349876404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3584,0.026687999566396076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,3072,0.02481813430786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2560,0.023422932624816893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,2048,0.021205333868662517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1536,0.019242666165033975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,768,0.01788160006205241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,512,0.017127466201782227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,1024,0.018295466899871826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,256,0.016225066781044007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,64,0.015863466262817382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,128,0.01612160007158915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,32,0.016059733430544534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,3072,65536,0.28504746754964194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,8192,0.040508798758188885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,12288,0.06273813247680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,10240,0.055795200665791835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,16384,0.06983253161112467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,7168,0.03741653362909953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,5120,0.03139413396517436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,6144,0.03598399957021077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,4096,0.027526400486628216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3584,0.025655466318130492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,3072,0.024437334140141806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2560,0.022530132532119752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,2048,0.020114133755366005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1536,0.019233065843582153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,1024,0.017729065815607705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,768,0.017612799008687337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,512,0.016936532656351724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,256,0.016060800353686015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,128,0.015598932902018229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,65536,0.264027738571167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,32,0.015893333156903586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2560,64,0.01573013365268707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,8192,0.03986133337020874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,10240,0.05065493186314901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,12288,0.05893653233846029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,16384,0.07064853509267172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,7168,0.0366048018137614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,6144,0.03395093282063802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,5120,0.030298666159311934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,4096,0.0270687997341156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2560,0.021655466159184775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3584,0.024819199244181314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,3072,0.023538132508595787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,2048,0.02026240030924479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1536,0.018874667088190713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,1024,0.01755839983622233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,768,0.01718719998995463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,512,0.016454399625460307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,65536,0.23659520149230956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,256,0.015920000274976094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,64,0.015530666708946228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,128,0.015473066767056783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,2048,32,0.015596800049146018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,10240,0.04721173445383708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,8192,0.03827519814173381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,12288,0.0553546667098999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,16384,0.06830080350240073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,5120,0.0288480003674825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,7168,0.03536746501922607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,6144,0.03253333369890849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,4096,0.02625280022621155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3584,0.02435520092646281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,3072,0.02203413248062134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2048,0.019973333676656088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,2560,0.021211733420689903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1536,0.018339200814565023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,1024,0.0170741339524587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,65536,0.21449813842773438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,768,0.016706132888793947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,512,0.016267733772595723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,128,0.015383467078208923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,256,0.015505066514015198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,64,0.015345066785812378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1536,32,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,12288,0.04798826773961385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,10240,0.04121813376744588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,8192,0.034731733798980716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,16384,0.061800531546274816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,5120,0.026774400472640993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,4096,0.023065600792566934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,6144,0.029577600955963134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,7168,0.03169813354810079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3072,0.021336533625920615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2048,0.018866133689880372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,3584,0.022180267175038657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,2560,0.019643733898798622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,65536,0.19084266026814778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1536,0.01768640081087748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,1024,0.016669867436091106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,768,0.016193067034085594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,512,0.01591679950555166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,256,0.015396266182263692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,128,0.015198933084805808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,64,0.015262933572133383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,1024,32,0.015225600202878317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,16384,0.0549888014793396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,8192,0.03202880024909973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,10240,0.03676799933115642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,12288,0.04216426610946655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,7168,0.029406932989756267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,4096,0.023570134242375692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,5120,0.025455999374389648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,6144,0.027616000175476073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3584,0.021822933355967203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,3072,0.021222400665283202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,65536,0.1831434726715088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2560,0.020029866695404054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,2048,0.01923840045928955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1024,0.016875733931859337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,1536,0.01755946675936381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,768,0.016429866353670754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,256,0.015280000368754067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,512,0.0162282665570577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,128,0.015052800377209982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,64,0.014986667037010192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,768,32,0.015122133493423461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,16384,0.05308053493499756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,12288,0.04145173231760661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,8192,0.030982400973637896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,10240,0.03679146766662598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,7168,0.029156267642974854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,6144,0.027074132363001508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,5120,0.024872533480326333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,4096,0.022946133216222128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3584,0.02202026645342509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,65536,0.1744277318318685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,3072,0.02068906625111898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2048,0.018588799238204955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,2560,0.019819732507069907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1536,0.017427200078964235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,1024,0.016639999548594155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,768,0.01606613298257192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,512,0.01570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,128,0.015092266599337259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,256,0.015155200163523355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,32,0.015020799636840821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,512,64,0.015025066335995993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,12288,0.0368127981821696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,16384,0.04505813519159953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,8192,0.030379732449849445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,10240,0.034125868479410806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,4096,0.02227733333905538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,6144,0.02584106723467509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,5120,0.024142932891845704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,7168,0.028663466374079388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,65536,0.16165013313293458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3584,0.02139413356781006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,3072,0.020361600319544475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2560,0.01893226703008016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,2048,0.018281600872675576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1536,0.017091200749079386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,1024,0.016671999295552572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,768,0.016100266575813295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,512,0.015658666690190635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,256,0.015270400047302245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,64,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,32,0.014869333306948344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,256,128,0.015030399958292643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,16384,0.04216426610946655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,10240,0.03262613415718078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,8192,0.0290613333384196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,12288,0.037013332049051925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,7168,0.027563732862472535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,6144,0.026228266954421996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,4096,0.021998933951059976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,5120,0.023915733893712363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,65536,0.15771199862162272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3584,0.02111146648724874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,3072,0.020090667406717937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2560,0.019113600254058838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,2048,0.018233599265416463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1536,0.01718399922053019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,1024,0.016547200083732606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,768,0.016229333480199178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,512,0.01565333306789398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,256,0.015118933717409768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,128,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,64,0.014939733346303306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,128,32,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,8192,0.028571732838948566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,12288,0.03286293347676595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,16384,0.04037439823150635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,10240,0.03111039996147156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,7168,0.0274399995803833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,4096,0.02193173368771871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,5120,0.023733333746592204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,6144,0.026023467381795246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,65536,0.15686933199564618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3584,0.020826667547225952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,3072,0.020052266120910645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2048,0.018075732390085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,2560,0.018823466698328652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1536,0.0170741339524587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,1024,0.01625706652800242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,512,0.015546666582425437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,768,0.016100266575813295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,256,0.014904533823331198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,128,0.014882133404413859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,64,0.014804266889890037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,64,32,0.01472106675306956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,10240,0.03100373347600301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,16384,0.03944426774978638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,8192,0.02831253409385681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,12288,0.033003733555475874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,5120,0.024025599161783852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,4096,0.021985065937042237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,7168,0.02727466622988383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,6144,0.02589226762453715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,65536,0.15486399332682294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2560,0.01893333395322164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3072,0.020227199792861937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,3584,0.02102400064468384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,2048,0.01798400084177653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1024,0.016267733772595723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,1536,0.017090133825937905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,768,0.01599679986635844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,256,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,512,0.01565439999103546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,128,0.015006933609644571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,64,0.01495146652062734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,512,32,32,0.014819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,7168,0.2668256123860677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,8192,0.30208212534586587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,10240,0.3756223996480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,12288,0.44170134862263993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,6144,0.23275626500447594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,5120,0.19927679697672526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,4096,0.16537386576334637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,16384,0.5989791870117187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,3584,0.15037333170572917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,1536,0.08787199656168619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,2048,0.09757973353068033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,2560,0.1159061352411906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,3072,0.13178346951802572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,1024,0.06861653327941894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,256,0.04354879856109619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,64,0.03882666826248169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,128,0.03894613186518352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,512,0.051481600602467856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,768,0.06033600171407064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,65536,32,0.040055465698242185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,8192,0.09281173547108969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,10240,0.1129642645517985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,12288,0.13042346636454266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,16384,0.16681706110636393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,5120,0.06423466602961223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,7168,0.08352533181508383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,6144,0.0740885337193807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,4096,0.05470186471939087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,3584,0.05006826718648275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,2048,0.03655360142389934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,2560,0.041262932618459064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,3072,0.045551999409993486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,1536,0.03153706590334575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,1024,0.026843732595443724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,768,0.025059199333190917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,512,0.023015467325846355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,256,0.020216532548268638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,128,0.01930346687634786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,64,0.01914773384730021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,32,0.019844265778859456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,10240,0.09009919961293539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,12288,0.10460373560587566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,16384,0.13214826583862305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,7168,0.06741759777069092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,8192,0.07429440021514892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,6144,0.059562667210896814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,16384,65536,0.648312505086263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,4096,0.045363199710845944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,5120,0.05230933427810669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,3584,0.041707734266916915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,2560,0.03466240167617798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,2048,0.031040000915527343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,1536,0.02776106595993042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,3072,0.04063040018081665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,512,0.021040000518163047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,768,0.02286293307940165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,1024,0.024195200204849242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,256,0.018692266941070557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,128,0.018279467026392618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,64,0.018387200435002644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,32,0.018810667594273887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,10240,0.08351466655731202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,12288,65536,0.49553813934326174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,12288,0.09672746658325196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,16384,0.12112639745076496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,6144,0.05625386635462443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,7168,0.06814826329549153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,5120,0.04952853520711263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,8192,0.0696938673655192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,3072,0.03659733136494954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,2560,0.033045333623886106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,3584,0.040516265233357746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,4096,0.04279786745707194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,2048,0.029009066025416058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,1536,0.026078933477401735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,768,0.021917865673700968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,1024,0.02327359914779663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,512,0.020182400941848755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,128,0.017760000626246133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,256,0.018542933464050292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,64,0.017886932690938315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,32,0.01774186690648397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,10240,0.06867413520812989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,10240,65536,0.44377492268880203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,12288,0.08015466531117757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,16384,0.10154773394266765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,6144,0.04761706590652466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,8192,0.05777173439661661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,7168,0.056150400638580324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,5120,0.04233173529307048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,2560,0.028586665789286297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,3584,0.033804798126220705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,3072,0.03116160035133362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,4096,0.036703999837239584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,1536,0.023877332607905068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,2048,0.02592533429463704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,1024,0.021492266654968263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,768,0.019973333676656088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,512,0.018602667252222697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,64,0.01721386710802714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,256,0.017670400937398276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,128,0.017171200116475424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,32,0.01726400057474772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,8192,65536,0.374344539642334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,12288,0.07590293089548747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,10240,0.065228799978892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,16384,0.09545173645019531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,8192,0.055555200576782225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,6144,0.04618026812871297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,7168,0.05070186853408813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,5120,0.040948267777760824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,4096,0.03623786767323812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,3584,0.03308586676915486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,2560,0.027963733673095702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,3072,0.030734932422637938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,2048,0.025577600797017413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,1536,0.023909332354863484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,1024,0.021321600675582884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,768,0.02050559918085734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,512,0.018326399723688762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,128,0.01753173271814982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,256,0.01816426714261373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,64,0.017638399203618368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,32,0.01765226721763611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,7168,65536,0.3511658668518066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,12288,0.07162559827168782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,16384,0.08668159643809001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,10240,0.0656768004099528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,8192,0.05275413195292154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,5120,0.03850880066553752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,7168,0.048479998111724855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,6144,0.04512213468551636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,4096,0.0342741330464681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,3584,0.03205759922663371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,3072,0.02972266674041748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,2048,0.02496853272120158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,2560,0.027051732937494917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,768,0.018657066424687705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,1536,0.02254613240559896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,512,0.017893334229787193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,1024,0.019992534319559732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,256,0.016989866892496742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,64,0.016645333170890807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,128,0.016544000307718913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,32,0.016703999042510985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,6144,65536,0.3206432024637858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,10240,0.05545813242594401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,12288,0.06416000127792358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,16384,0.07951680024464926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,8192,0.047839999198913574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,5120,0.03479573329289754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,6144,0.03959466616312663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,4096,0.030297599236170453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,7168,0.04351679881413777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,3584,0.028037333488464357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,2560,0.024639999866485594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,3072,0.026238934199015303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,2048,0.02278613249460856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,1536,0.02134400010108948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,1024,0.019942400852839152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,768,0.018412800629933675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,512,0.017323732376098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,256,0.016695467631022136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,64,0.016378666957219443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,128,0.016489600141843162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,32,0.01650879979133606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,5120,65536,0.27951361338297526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,10240,0.05824000040690104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,12288,0.06668266455332438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,8192,0.04331733385721843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,16384,0.07948479652404786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,5120,0.03129706581433614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,4096,0.028300799926122028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,7168,0.0399402658144633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,6144,0.036149334907531736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,2560,0.02313279906908671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,3584,0.025971200068791705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,3072,0.024681599934895833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,2048,0.02139413356781006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,1024,0.017899733781814576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,1536,0.019896533091862997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,512,0.01681386629740397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,768,0.017398399114608765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,128,0.015733333428700765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,64,0.015913599729537965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,256,0.016291200121243795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,65536,0.25113066037495935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,4096,32,0.015918933351834617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,8192,0.04232213497161865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,10240,0.049235200881958006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,12288,0.056287999947865805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,16384,0.06948800086975097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,7168,0.03907519976298014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,6144,0.03504746754964193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,5120,0.030482133229573567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,4096,0.02735466758410136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,3584,0.02592639923095703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,2560,0.02273599902788798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,3072,0.024421334266662598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,2048,0.021549866596857706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,1024,0.01823893388112386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,1536,0.019912532965342202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,768,0.017670400937398276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,512,0.017081600427627564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,256,0.01637333333492279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,128,0.015828266739845276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,64,0.01606933375199636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,65536,0.238265593846639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3584,32,0.016275200247764587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,10240,0.04599253336588542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,8192,0.039230934778849286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,12288,0.05250026782353719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,16384,0.06481493314107259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,7168,0.03546453317006429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,5120,0.028854399919509888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,6144,0.03221866687138875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,4096,0.0263264000415802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,3072,0.02332373261451721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,2048,0.020550400018692017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,3584,0.024796799818674723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,2560,0.0221343994140625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,1536,0.019317332903544107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,1024,0.01781439979871114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,768,0.017203199863433837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,512,0.01646506687005361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,65536,0.22519572575887045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,256,0.015988266468048094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,128,0.016140799721082053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,64,0.015811199943224587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,3072,32,0.015828266739845276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,8192,0.03589119911193848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,10240,0.043186132113138834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,12288,0.04965333143870036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,16384,0.06096426645914713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,6144,0.02990399996439616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,4096,0.024940800666809083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,7168,0.032525867223739624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,5120,0.027102933327356978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,2560,0.020963199933369956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,3072,0.022322134176890055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,3584,0.023461333910624185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,2048,0.019562667608261107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,1536,0.017949867248535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,65536,0.20612160364786783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,1024,0.017334399620691936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,768,0.016872533162434897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,512,0.016058666507403056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,256,0.01617280046145121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,64,0.01570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,128,0.015681067109107973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2560,32,0.01569919983545939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,10240,0.04193280140558879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,12288,0.052502401669820155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,8192,0.03439040184020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,16384,0.06499733527501425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,7168,0.03131519953409831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,6144,0.029883732398351032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,5120,0.02686079939206441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,4096,0.024336000283559166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,3584,0.021926399072011313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,3072,0.021163733800252278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,2560,0.019553067286809285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,2048,0.018768000602722167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,65536,0.18394239743550617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,1024,0.017205333709716795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,1536,0.01811520059903463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,512,0.01625920037428538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,768,0.016569599509239197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,128,0.015245866775512696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,256,0.015496533115704855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,64,0.015446399648984274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,2048,32,0.015590399503707886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,10240,0.03657493193944295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,16384,0.054330666859944664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,12288,0.04287679990132649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,8192,0.031939200560251874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,7168,0.029019733270009358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,6144,0.0272053341070811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,5120,0.02662186622619629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,4096,0.02438933253288269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,3584,0.02292693257331848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,3072,0.021658666928609214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,2560,0.01982933282852173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,2048,0.018922666708628334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,65536,0.169758939743042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,1536,0.017781333128611247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,1024,0.01707093318303426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,768,0.016531200210253397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,512,0.01595200002193451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,256,0.015559466679890952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,128,0.015296000242233276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,64,0.015221333503723145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1536,32,0.0151637335618337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,16384,0.048877867062886556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,10240,0.033557331562042235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,8192,0.029658667246500653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,12288,0.03739946683247884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,7168,0.02775146762530009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,4096,0.02218986749649048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,6144,0.02574719985326131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,5120,0.02474453250567118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,3584,0.021291732788085938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,65536,0.15908800760904948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,3072,0.020542933543523153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,2560,0.019314134120941163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,2048,0.018687999248504637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,1536,0.017683200041453042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,768,0.016501333316167197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,512,0.01588159998257955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,1024,0.01672853430112203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,256,0.01532906691233317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,128,0.014998400211334228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,64,0.015054933230082192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,1024,32,0.0151146670182546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,16384,0.046649599075317384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,8192,0.029575467109680176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,10240,0.03327786723772685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,12288,0.03722773392995198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,7168,0.0271071990331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,5120,0.02323626677195231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,6144,0.025758934020996094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,4096,0.021562665700912476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,3584,0.020643200476964316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,65536,0.14535147349039715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,2560,0.01880853374799093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,3072,0.0200053334236145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,2048,0.018068265914916993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,1536,0.01744426687558492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,1024,0.01681813398996989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,768,0.016448000073432924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,512,0.015946666399637856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,128,0.015077333648999533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,256,0.015406933426856995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,64,0.015025066335995993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,768,32,0.0150026669104894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,8192,0.027852799495061236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,16384,0.042045867443084715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,12288,0.03535573482513428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,10240,0.031593600908915206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,7168,0.026676267385482788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,6144,0.02472320000330607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,5120,0.02308373252550761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,4096,0.021653334299723305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,65536,0.13728000322977701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,3584,0.02030613422393799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,3072,0.019662932554880778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,2560,0.018918399016062418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,2048,0.01787839929262797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,1536,0.017378133535385133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,1024,0.016245333353678386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,768,0.016049066185951234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,512,0.015458133816719056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,128,0.014942933122316995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,256,0.01518933375676473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,64,0.014916266997655234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,512,32,0.01486186683177948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,16384,0.03789226611455281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,12288,0.03155200084050496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,10240,0.03020799954732259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,8192,0.027445334196090698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,7168,0.025483733415603636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,4096,0.020794665813446044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,5120,0.02209279934565226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,6144,0.024013866980870567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,65536,0.1272981325785319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,3584,0.019870932896931967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,2560,0.01818880041440328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,3072,0.019042134284973145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,2048,0.017640533049901326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,1536,0.01689493258794149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,1024,0.01629973351955414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,768,0.01598186691602071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,256,0.015241600076357522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,512,0.015426133076349893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,128,0.01495253344376882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,64,0.014944000045458474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,256,32,0.01488746702671051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,12288,0.030426667133967085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,16384,0.03638186852137248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,8192,0.02587946653366089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,10240,0.02850773334503174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,7168,0.025244800249735515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,4096,0.02071466644605001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,5120,0.022395733992258707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,6144,0.024035199483235677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,65536,0.12492586771647136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,3584,0.019853866100311278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,3072,0.01908586621284485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,2560,0.018216532468795777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,1536,0.01686186591784159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,2048,0.01754986643791199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,1024,0.016109866897265117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,768,0.015995732943216958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,256,0.015053866306940713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,512,0.015511467059453329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,128,0.014798933267593383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,64,0.01479573349157969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,128,32,0.014845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,16384,0.03504960139592488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,12288,0.029257599512736005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,10240,0.02792106668154399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,8192,0.025918932755788167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,7168,0.025219200054804485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,5120,0.022181334098180135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,4096,0.020665599902470907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,6144,0.023910399278004965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,65536,0.12360213597615559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,3584,0.01963520050048828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,2560,0.018212266763051353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,3072,0.01917546590169271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,2048,0.017386666933695474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,1536,0.01699413259824117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,1024,0.016179200013478598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,768,0.01600106656551361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,512,0.015340800086657206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,128,0.014904533823331198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,256,0.015042133132616677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,32,0.014737066626548768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,64,64,0.014758400122324624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,12288,0.02932586669921875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,16384,0.033318400382995605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,8192,0.025682133436203004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,10240,0.027932800849278766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,7168,0.024991999069849648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,6144,0.024103466669718424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,4096,0.020617600282033285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,5120,0.02215999960899353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,65536,0.12007679939270019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,3584,0.019513599077860513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,2560,0.017977599302927652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,3072,0.018921599785486857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,2048,0.01767680048942566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,1536,0.016774400075276693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,1024,0.01600106656551361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,768,0.015868799885114034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,256,0.015146666765213012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,128,0.014788267016410828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,512,0.015336533387502035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,64,0.014788267016410828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,384,32,32,0.014717866977055868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,7168,0.20829547246297203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,8192,0.2328202724456787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,10240,0.2809290568033854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,12288,0.3320437431335449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,5120,0.15933440526326498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,6144,0.1842613379160563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,4096,0.1327946662902832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,16384,0.42773758570353193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3584,0.12078399658203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2048,0.07946773370107016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,2560,0.09392639795939127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,3072,0.10831466515858967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1536,0.07260586420694987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,1024,0.053846398989359535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,256,0.03494826555252075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,128,0.030517333745956422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,512,0.04206826686859131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,768,0.04777919848759969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,64,0.029173332452774047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,65536,32,0.032529066006342575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,8192,0.07579413255055746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,10240,0.09043413003285726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,12288,0.10433493455251057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,16384,0.13027199904123943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,5120,0.05383679866790771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,6144,0.061292799313863125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,7168,0.06848853429158529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,4096,0.04681066672007243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2560,0.035997867584228516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,2048,0.031598933537801105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3072,0.03999786774317424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,3584,0.04355306625366211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,512,0.020626133680343627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1024,0.023539199431737264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,1536,0.026985599597295122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,768,0.022094933191935222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,256,0.018244266510009766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,128,0.017426133155822754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,64,0.017837866147359212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,32,0.018308266003926595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,10240,0.07363306681315104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,12288,0.08586666584014893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,16384,0.10572693347930909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,7168,0.05658986568450928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,8192,0.062393601735432944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,6144,0.0510047992070516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,16384,65536,0.46242666244506836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,4096,0.03966720104217529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,5120,0.0451200008392334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3584,0.03701759974161784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,3072,0.036297599474589035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2560,0.030580266316731768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,2048,0.026606933275858564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1536,0.02387626568476359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,1024,0.02193066676457723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,768,0.020824533700942994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,512,0.018607999881108603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,256,0.01744426687558492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,128,0.016839466492335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,64,0.01714026729265849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,32,0.01763733426729838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,10240,0.0666101336479187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,12288,0.0767626682917277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,12288,65536,0.35871998469034827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,16384,0.09528533617655435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,5120,0.04145493507385254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,8192,0.055929601192474365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,7168,0.05161386728286743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,6144,0.0504095991452535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3072,0.031056000789006548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2560,0.028114134073257448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,4096,0.036370134353637694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,3584,0.03463893334070842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,2048,0.025538132588068647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1536,0.023537067572275798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,1024,0.021505065759023032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,768,0.020619734128316244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,512,0.018156800667444864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,256,0.01765759984652201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,128,0.01691626707712809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,64,0.016884267330169678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,32,0.01710933248202006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,10240,65536,0.35064001083374025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,12288,0.06621546745300293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,10240,0.05712106625239054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,16384,0.08231573104858399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,7168,0.04474026759465535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,8192,0.05289920171101888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,5120,0.036414933204650876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,6144,0.04109013477961222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3072,0.02691306670506795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,3584,0.02953706582387288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2560,0.025026132663091023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,4096,0.032151466608047484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1536,0.02134079933166504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,2048,0.023135999838511147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,1024,0.020004266500473024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,768,0.01849386692047119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,512,0.01704746683438619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,256,0.01665493349234263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,128,0.016311466693878174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,64,0.016448000073432924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,32,0.01694933374722799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,8192,65536,0.26818774541219076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,10240,0.05324480136235556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,12288,0.060821334520975746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,16384,0.07551786899566651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,8192,0.04535359938939412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,7168,0.04150506655375163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,4096,0.02948906620343526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,5120,0.03419093290964763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,6144,0.03810453414916992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3584,0.02714560031890869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,3072,0.02546773354212443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2560,0.02408213416735331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,2048,0.022525866826375328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1536,0.020666666825612388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,1024,0.01930346687634786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,768,0.017335466543833413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,512,0.016953599452972413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,256,0.016502400239308677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,128,0.016225066781044007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,64,0.016145066420237223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,32,0.016482133666674295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,7168,65536,0.24542080561319984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,10240,0.04974186817804972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,12288,0.05674773454666138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,16384,0.06971200307210287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,8192,0.04221333265304565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,4096,0.027356799443562823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,5120,0.03166933258374532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,7168,0.0387722651163737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,6144,0.03576533397038777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3584,0.025857067108154295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,3072,0.02426346739133199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2560,0.02312320073445638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,2048,0.02180373271306356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1536,0.020126932859420778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,1024,0.01775040030479431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,768,0.018155733744303383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,512,0.016546133160591125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,256,0.016286933422088624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,128,0.015745066603024802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,64,0.015960533420244852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,32,0.016178133090337117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,6144,65536,0.23503680229187013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,12288,0.052817066510518394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,8192,0.03953280051549275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,10240,0.047386666138966874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,16384,0.06338560183842977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,6144,0.033655468622843424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,5120,0.029703466097513835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,4096,0.026385066906611125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,7168,0.0364682674407959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3584,0.02507200042406718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,3072,0.02372480034828186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2560,0.022397865851720176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,2048,0.020965333779652914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1536,0.01986453334490458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,1024,0.017753599087397258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,768,0.01711146632830302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,512,0.016535466909408568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,256,0.01609173317750295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,128,0.015701333681742348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,65536,0.21274666786193847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,64,0.0157642662525177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,5120,32,0.016194132963816325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,16384,0.05999039808909098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,12288,0.049907199541727704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,8192,0.037562668323516846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,10240,0.0454645315806071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,7168,0.03487039804458618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,6144,0.031173332532246904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,5120,0.028421332438786823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,4096,0.025690666834513348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3584,0.024337067206700643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,3072,0.02277440031369527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2560,0.021720532576243082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,2048,0.02018773357073466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1536,0.018321067094802856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,1024,0.017383466164271034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,768,0.01686933239301046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,512,0.01632213294506073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,65536,0.18478186925252277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,256,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,128,0.01532586713631948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,64,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,4096,32,0.015892266233762106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,8192,0.03557973305384318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,10240,0.04066986640294393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,12288,0.04605120023091634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,16384,0.05620479981104533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,7168,0.03230080008506775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,6144,0.029757867256800335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,5120,0.026485333840052288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,4096,0.024707200129826863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3072,0.022218666474024453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2560,0.020787199338277183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,3584,0.023113600413004556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,2048,0.019801600774129232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1024,0.017027199268341064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,1536,0.019080533583958944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,768,0.016377600034077962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,65536,0.17658027013142902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,512,0.016029866536458333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,256,0.015683199961980185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,64,0.015459199746449789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,128,0.01498133341471354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3584,32,0.015719466408093772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,8192,0.03317973415056864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,10240,0.03979733387629191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,12288,0.04455466667811076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,16384,0.054218665758768714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,7168,0.030374399820963544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,6144,0.02826346755027771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,4096,0.023746132850646973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,5120,0.025781333446502686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3584,0.02235413392384847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,3072,0.021348265806833903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2560,0.020295466979344687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,2048,0.01926506757736206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1536,0.017867734034856163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,65536,0.16386027336120607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,1024,0.01665279964605967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,768,0.01623040040334066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,256,0.01551253298918406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,512,0.015783466895421348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,128,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,32,0.015596800049146018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,3072,64,0.015335466464360556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,12288,0.03976213137308757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,10240,0.03760319948196411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,8192,0.03166399995485942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,16384,0.04957120021184285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,7168,0.028930133581161498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,6144,0.026842667659123735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,5120,0.02481813430786133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,4096,0.022950400908788048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3584,0.02187733252843221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,3072,0.020795732736587524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2560,0.01991466681162516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,2048,0.018489599227905273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,65536,0.15330346425374347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1536,0.017126399278640746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,1024,0.016472533345222473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,768,0.016057599584261575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,512,0.015828266739845276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,256,0.015520000457763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,128,0.015152000387509666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,64,0.015154133240381876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2560,32,0.015296000242233276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,12288,0.039510401089986165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,10240,0.03461546500523885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,16384,0.04801493485768636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,8192,0.029769599437713623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,7168,0.027780266602834065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,6144,0.025629866123199462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,5120,0.023733333746592204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,4096,0.022230400641759237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3072,0.020152533054351808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,3584,0.02085226575533549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2560,0.017784533898035686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,65536,0.13730667432149252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,2048,0.0180949330329895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1536,0.017029333114624023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,1024,0.016396799683570863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,768,0.015954132874806723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,512,0.015733333428700765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,256,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,128,0.015202132860819497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,64,0.015280000368754067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,2048,32,0.015541332960128783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,12288,0.03397013346354167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,10240,0.030359466870625813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,8192,0.02757973273595174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,16384,0.04192853371302287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,5120,0.02255679965019226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,6144,0.024102399746576943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,7168,0.02545706629753113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,4096,0.020333866278330483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,65536,0.12496319611867268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3584,0.019594667355219524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,3072,0.019332265853881835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2560,0.01839359998703003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,2048,0.01755733291308085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1024,0.016948266824086507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,1536,0.01704960068066915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,512,0.015803733468055726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,768,0.01602453291416168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,256,0.0150218665599823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,128,0.014987732966740927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,64,0.014983466267585755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1536,32,0.015247999628384908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,16384,0.04003200133641561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,12288,0.03335573275883992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,10240,0.03127999901771546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,8192,0.026343466838200887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,6144,0.023560533920923867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,5120,0.022325332959493002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,7168,0.02513493299484253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,4096,0.020441599686940513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,65536,0.11099627017974853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3584,0.019654399156570433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,3072,0.01889280080795288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2560,0.018266665935516357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,2048,0.01762239933013916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1536,0.01685653328895569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,1024,0.016475733121236166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,768,0.015758933623631795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,512,0.015317333738009134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,256,0.01516480048497518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,128,0.01495680014292399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,64,0.014966400464375815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,1024,32,0.01492693324883779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,16384,0.0371509313583374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,10240,0.028642133871714277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,12288,0.03173759977022807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,8192,0.025702399015426636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,7168,0.023806933561960855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,5120,0.021333332856496176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,6144,0.022539732853571574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,65536,0.10504106680552165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,4096,0.019884800910949706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3584,0.019015467166900633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1536,0.016774400075276693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2048,0.017545600732167564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,3072,0.018606932957967122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,2560,0.018242132663726807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,1024,0.01636693378289541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,768,0.01593706707159678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,128,0.014980266491572062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,512,0.015359999736150107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,256,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,64,0.01492800017197927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,768,32,0.015064533551534018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,12288,0.029260800282160444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,16384,0.03360533316930135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,10240,0.026973867416381837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,8192,0.024915200471878052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,6144,0.022742400566736855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,7168,0.02344533403714498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,65536,0.09865067005157471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,5120,0.020987733205159505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,4096,0.019581866264343262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3584,0.0191103994846344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,3072,0.018203733364741008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2560,0.01775253415107727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,2048,0.01722773313522339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1536,0.016661333044370015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,1024,0.016134400169054666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,768,0.01567146678765615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,512,0.015362133582433065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,256,0.015092266599337259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,128,0.01511146624883016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,64,0.014732799927393594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,512,32,0.014825600385665893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,16384,0.02990613381067912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,12288,0.02952959934870402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,8192,0.02437439958254496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,10240,0.02617173393567403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,7168,0.023253333568573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,65536,0.09150400161743164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,6144,0.02225066622098287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,5120,0.020866133769353232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,4096,0.019554134209950766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3584,0.01880106727282206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,3072,0.018216532468795777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2048,0.017072000106175742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,2560,0.0178656001885732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1536,0.01633280018965403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,1024,0.016075733304023742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,768,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,512,0.015334399541219077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,256,0.015093333522478738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,128,0.014907733599344889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,64,0.014856533209482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,256,32,0.015044266978899637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,16384,0.02898453275362651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,12288,0.025755733251571655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,8192,0.02358293334643046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,10240,0.024850134054819742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,65536,0.08953279654184977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,7168,0.023254400491714476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,6144,0.02216213345527649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,4096,0.019282132387161255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,5120,0.020683733622233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3584,0.018706132968266807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,3072,0.01813760002454122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2560,0.017590399583180746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,2048,0.017052799463272095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1536,0.016495999693870545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,1024,0.016039466857910155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,768,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,128,0.01471466620763143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,512,0.015188266833623251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,256,0.01498240033785502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,32,0.014802133043607077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,128,64,0.014831999937693277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,12288,0.025921066602071125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,16384,0.028590933481852217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,8192,0.023362133900324503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,10240,0.025154133637746174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,65536,0.0860053300857544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,7168,0.023146667083104453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,6144,0.022102399667104086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,5120,0.020826667547225952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3584,0.01869759956995646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,3072,0.018097066879272462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,4096,0.019357866048812865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2560,0.01751040021578471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,2048,0.016871466239293417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1536,0.016506666938463845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,1024,0.015957333644231162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,768,0.015574399630228677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,512,0.015280000368754067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,256,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,128,0.01482133368651072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,64,0.01470186710357666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,64,32,0.014852266510327658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,16384,0.02835200031598409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,12288,0.025784534215927125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,10240,0.025029333432515462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,8192,0.023059199253718056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,65536,0.08359253406524658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,7168,0.023085866371790567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,5120,0.020451200008392335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,6144,0.02205866575241089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,4096,0.01924053430557251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3584,0.018348799149195353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,3072,0.01816640098889669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2560,0.017578667402267455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,2048,0.0172160009543101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1536,0.016379732886950174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,1024,0.015675733486811318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,768,0.015495466192563376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,512,0.015294933319091797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,256,0.014852266510327658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,128,0.01477226714293162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,64,0.014577066898345948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,256,32,32,0.014698666334152222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,7168,0.1989397366841634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,8192,0.21987412770589193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,10240,0.26539093653361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,12288,0.3135349273681641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,5120,0.15018240610758465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,6144,0.17637866338094074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,4096,0.12597119808197021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,16384,0.40434878667195634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3584,0.11498239835103352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1024,0.05268693367640177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2560,0.09139520327250163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,3072,0.10302186806996663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,2048,0.07682027022043864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,1536,0.07124906380971273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,64,0.027880533536275225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,256,0.031931734085083006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,128,0.02804479996363322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,768,0.047261865933736165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,512,0.03760639826456706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,65536,32,0.02942613363265991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,8192,0.06988373597462973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,10240,0.08344746430714925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,12288,0.09624319871266683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,16384,0.1202624003092448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,7168,0.0633407990137736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,6144,0.05745919942855835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,5120,0.0507370670636495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,4096,0.04427093267440796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3584,0.04171093304951985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2560,0.03424319823582967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,3072,0.037829331556955975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,2048,0.030033065875371294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1024,0.022447999318440756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,1536,0.026133332649866742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,768,0.021237333615620933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,512,0.01986666719118754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,256,0.017439999183019004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,128,0.017129600048065186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,64,0.017500799894332886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,32,0.01801919937133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,12288,0.07884159882863363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,10240,0.06837120056152343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,16384,0.09833813508351644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,7168,0.05303039948145548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,8192,0.05772159894307455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,6144,0.04815253416697184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,16384,65536,0.42110293706258134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,4096,0.03741653362909953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3584,0.035233068466186526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,5120,0.04272533257802327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,3072,0.03464106718699138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2048,0.025763199726740522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,2560,0.028546132644017536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1536,0.023167999585469563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,1024,0.020983467499415077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,768,0.01990506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,512,0.01798506577809652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,128,0.016516266266504924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,256,0.01693120002746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,32,0.017026132345199584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,64,0.016684800386428833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,12288,65536,0.3553322792053223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,10240,0.06132053136825562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,12288,0.07023893197377523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,16384,0.08742079734802247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,7168,0.048341333866119385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,8192,0.057145599524180093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,6144,0.04370239973068237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,5120,0.039214932918548585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3072,0.02954453428586324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2560,0.026817067464192705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,3584,0.03237866759300232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,4096,0.03445013364156087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,2048,0.024995199839274087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1024,0.020600533485412596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,768,0.019595734278361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,1536,0.022894932826360067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,512,0.0173632005850474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,256,0.01672533353169759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,128,0.016345600287119545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,64,0.016676266988118492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,32,0.017191465695699057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,10240,65536,0.3143967946370443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,12288,0.06146133343378703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,10240,0.05508373181025187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,16384,0.07598079840342203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,8192,0.0492789347966512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,7168,0.04158186515172323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,5120,0.03442879915237427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,6144,0.03813013235727946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3072,0.026051199436187743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2560,0.024053333202997844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,4096,0.030187733968098956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,3584,0.027734400828679402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,2048,0.022392533222834268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1536,0.02061333258946737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,1024,0.019108267625172932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,768,0.017697066068649292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,512,0.016807466745376587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,128,0.015633066495259605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,64,0.01613653302192688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,256,0.01606186628341675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,32,0.01656426688035329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,8192,65536,0.25607892672220867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,12288,0.0602570652961731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,10240,0.053650132815043126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,16384,0.07181013425191243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,8192,0.04289493163426717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,7168,0.03932266632715861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,5120,0.03264000018437703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,6144,0.03585919936498006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,4096,0.02800533374150594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2560,0.02410986622174581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3072,0.026060799757639568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,3584,0.0276202658812205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,2048,0.022487467527389525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1024,0.019036799669265747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,1536,0.02076906760533651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,768,0.01688533425331116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,512,0.0170741339524587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,128,0.01602453291416168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,256,0.016438399751981102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,32,0.016391467054684958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,64,0.01617173353830973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,7168,65536,0.24377172787984214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,10240,0.05132906834284464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,12288,0.055785600344340006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,8192,0.042683732509613034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,16384,0.06665813525517782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,5120,0.03157973289489746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,6144,0.035955198605855304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,7168,0.03865493138631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,4096,0.027852799495061236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3584,0.026172800858815508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,3072,0.02471253275871277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2560,0.023678932587305704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,2048,0.021338667472203574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1536,0.020194133122762047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,1024,0.017553067207336424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,768,0.01700586676597595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,512,0.016406400005022685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,256,0.015990400314331056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,128,0.015564800302187601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,64,0.015748266379038492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,32,0.015827199816703795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,6144,65536,0.20914667447408042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,12288,0.049407998720804855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,10240,0.04336426655451457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,8192,0.037835733095804853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,16384,0.060329600175221765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,5120,0.029066665967305498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,4096,0.02603093385696411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,6144,0.0317738672097524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,7168,0.035595734914143876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3584,0.024627200762430825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,3072,0.023144533236821495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2560,0.021944532791773476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,2048,0.02053546706835429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1536,0.019258666038513183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,1024,0.01848213275273641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,512,0.016289066274960837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,768,0.016985599199930826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,256,0.016198399662971496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,65536,0.187662935256958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,128,0.015624533096949259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,64,0.015480533242225647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,5120,32,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,10240,0.039868799845377605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,12288,0.048026665051778154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,8192,0.033137067159016924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,16384,0.057810131708780924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,7168,0.030230400959650676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,6144,0.0288480003674825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,4096,0.02378986676534017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,5120,0.02573653260866801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2560,0.02070080041885376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3584,0.022437334060668945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,3072,0.021564799547195434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,2048,0.019504000743230186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1536,0.0191594660282135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,1024,0.01660586694876353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,768,0.016657066345214844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,512,0.016082132856051125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,65536,0.16423786481221517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,256,0.01575040022532145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,128,0.015229866902033488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,64,0.015473066767056783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,4096,32,0.015702399611473083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,8192,0.03557973305384318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,12288,0.04354986747105916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,10240,0.03942399819691976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,16384,0.05385173161824545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,7168,0.03167999982833862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,6144,0.03009066581726074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,5120,0.026284799973169966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,4096,0.024387200673421226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3072,0.021847466627756752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2560,0.020654932657877604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,2048,0.01989333430926005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,3584,0.022380799055099487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,65536,0.1549898624420166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1536,0.017883733908335368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,1024,0.01699413259824117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,768,0.01625706652800242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,512,0.01599679986635844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,256,0.015785599748293556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,128,0.015389866630236306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,64,0.015434666474660238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3584,32,0.01544319987297058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,10240,0.036820268630981444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,8192,0.03181973298390706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,12288,0.040454399585723874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,16384,0.051478401819864905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,6144,0.028809599081675213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,7168,0.027077333132425947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,5120,0.02576533357302348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,4096,0.023319466908772787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3584,0.022411733865737915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,3072,0.021358933051427206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2048,0.018438400824864705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,2560,0.02036799987157186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,65536,0.14206825892130534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1024,0.016749866803487144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,1536,0.01777600049972534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,768,0.01632426679134369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,512,0.015999999642372132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,256,0.015541332960128783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,128,0.015265066425005594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,32,0.01550933321317037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,3072,64,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,10240,0.032553599278132124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,12288,0.03686399857203166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,16384,0.04498773415883382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,8192,0.02856106758117676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,7168,0.026734934250513716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,5120,0.02329279979070028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,6144,0.025183999538421632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,4096,0.021607466538747153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3584,0.020986666282018028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2560,0.01891626715660095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,3072,0.020773333311080933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,2048,0.017781333128611247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,65536,0.12946133613586425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1536,0.016976000865300496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,1024,0.016293332974116007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,768,0.01627626617749532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,512,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,256,0.015335466464360556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,128,0.015197867155075073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,64,0.015135999520619711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2560,32,0.015324800213177999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,12288,0.033218133449554446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,16384,0.040839465459187825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,10240,0.029483733574549358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,8192,0.028516266743342084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,7168,0.026473599672317504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,6144,0.024757333596547446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,5120,0.023915733893712363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,4096,0.02220373352368673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3584,0.020713599522908528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,3072,0.020039467016855876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,65536,0.11599893569946289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2560,0.01876693367958069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,2048,0.018260266383488974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1536,0.01770240068435669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,1024,0.01654293338457743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,768,0.016166399916013083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,512,0.016012799739837647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,256,0.015254400173823037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,128,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,32,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,2048,64,0.015165866414705912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,12288,0.03175786733627319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,10240,0.028961066404978437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,16384,0.03920746644337972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,8192,0.025940267244974773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,7168,0.024592000246047973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,6144,0.023125332593917847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,5120,0.021523199478785195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,4096,0.02027946710586548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,65536,0.10482347011566162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2560,0.018182400862375894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,2048,0.01757546663284302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3584,0.019949867328008016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,3072,0.01821440060933431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1536,0.01685973405838013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,1024,0.01600320041179657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,512,0.015650133291880287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,768,0.015918933351834617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,256,0.015293866395950317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,128,0.015122133493423461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,32,0.015118933717409768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1536,64,0.015050666530927024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,12288,0.02974613308906555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,16384,0.034220798810323076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,8192,0.024405332406361897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,10240,0.026731733481089277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,7168,0.023124267657597862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,6144,0.021772799889246623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,65536,0.09955093065897623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,5120,0.02035306692123413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,4096,0.01928960084915161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3584,0.01817173361778259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,3072,0.017942400773366292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2560,0.017672532796859743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1536,0.016752000649770102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,2048,0.017190400759379068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,1024,0.01600213348865509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,768,0.01569386621316274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,256,0.015024000406265258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,512,0.015477333466211954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,128,0.01507306694984436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,64,0.01493333379427592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,1024,32,0.014889599879582724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,16384,0.033123199144999185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,10240,0.02701866626739502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,12288,0.02951253255208333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,8192,0.02447893420855204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,6144,0.021746132771174112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,7168,0.022832000255584718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,5120,0.020267732938130698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,65536,0.08434560298919677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,4096,0.01904746691385905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3072,0.018054399887720743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,3584,0.018331732352574667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2048,0.01700693368911743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,2560,0.017849600315093993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1536,0.016433067123095193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,1024,0.015893333156903586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,768,0.015565866231918335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,512,0.015524267156918844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,256,0.015027200182278952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,128,0.014878933628400167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,64,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,768,32,0.014898133277893067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,16384,0.030305065711339313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,12288,0.027029333511988322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,10240,0.025110399723052977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,8192,0.023062400023142495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,65536,0.07871786753336588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,7168,0.022316799561182658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,6144,0.020932267109553017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,5120,0.01990506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,4096,0.018973867098490395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3584,0.018583466609319053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,3072,0.01806186636288961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2560,0.017425066232681273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,2048,0.016758400201797485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1536,0.01660693287849426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,1024,0.01596799989541372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,768,0.01556373337904612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,128,0.014892799655596414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,256,0.015058133006095886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,512,0.015209600329399109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,64,0.014914133151372275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,512,32,0.014808533589045205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,16384,0.028347732623418172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,12288,0.024920533100763954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,10240,0.023285333315531412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,8192,0.022105600436528525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,65536,0.07339413166046142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,7168,0.022091732422510783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,5120,0.01996906598409017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,6144,0.02109760046005249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,4096,0.018931200106938682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3584,0.0185098667939504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2560,0.01750826636950175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,3072,0.017965867122014364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1536,0.01647040049235026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,2048,0.01662399967511495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,1024,0.015672533710797628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,768,0.015572266777356467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,512,0.015324800213177999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,256,0.014932266871134438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,128,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,64,0.014792533715566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,256,32,0.014750933647155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,12288,0.023792000611623128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,10240,0.023491199811299643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,16384,0.026281599203745527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,8192,0.022156800826390585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,65536,0.06944746971130371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,6144,0.021185066302617392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,7168,0.02193386753400167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,5120,0.019797333081563315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,4096,0.01882560054461161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3584,0.018201599518458046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,3072,0.017787732680638633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1536,0.016239999731381734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2560,0.017195733388264973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,2048,0.016899200280507405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,1024,0.01584106683731079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,768,0.015468800067901611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,512,0.01520746648311615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,128,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,256,0.014988799889882406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,32,0.014763733744621277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,128,64,0.014823466539382935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,12288,0.023792000611623128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,16384,0.026332799593607587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,8192,0.022130133708318074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,10240,0.023310933510462442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,65536,0.06478506724039713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,7168,0.02175253431002299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,6144,0.021042132377624513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,4096,0.01876693367958069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,5120,0.01966080069541931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3584,0.018399999539057414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2048,0.016860800981521606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,3072,0.017873066663742065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,2560,0.017500799894332886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1536,0.016309332847595216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,1024,0.015812266866366068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,768,0.015549866358439126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,512,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,128,0.014762666821479798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,256,0.014852266510327658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,64,0.014814933141072592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,64,32,0.014759467045466105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,10240,0.02339199980099996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,16384,0.025986133019129436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,12288,0.024014933904012045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,8192,0.02211946646372477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,65536,0.0618229349454244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,7168,0.021618133783340453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,6144,0.02119040091832479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,5120,0.019637332359949747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,4096,0.018846933046976724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3584,0.01808639963467916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,3072,0.017893334229787193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2560,0.01723519961039225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,2048,0.01673813263575236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1536,0.016321067015329996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,1024,0.015446399648984274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,768,0.01567893326282501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,512,0.0152319997549057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,256,0.014860799908638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,128,0.01481066644191742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,32,0.014707199732462563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,192,32,64,0.014552533626556396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,7168,0.18906240463256835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,8192,0.21518826484680176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,10240,0.2605013370513916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,12288,0.30771627426147463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,5120,0.1425674597422282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,6144,0.16582400004069012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,4096,0.11939093271891277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,16384,0.4030495961507161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3584,0.10745066801706951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2048,0.07174293200174967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,2560,0.08422933419545492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1024,0.04746026595433553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,3072,0.0961301326751709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,1536,0.06537066698074341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,256,0.026700800657272337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,128,0.02323413292566935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,64,0.022733867168426514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,512,0.03410880168279012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,768,0.04127360184987386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,65536,32,0.02249280015627543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,8192,0.0679263989130656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,10240,0.08013439973195394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,12288,0.09310826460520426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,16384,0.11655360062917072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,6144,0.05557119846343994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,5120,0.04958933194478353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,7168,0.06196159919102987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,4096,0.043315200010935466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3072,0.03693866729736328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2560,0.03377279837926229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,3584,0.04015680154164632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,2048,0.029319467147191365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1024,0.022124799092610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,1536,0.025549866755803424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,768,0.020870399475097657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,512,0.019296000401178993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,256,0.01740586757659912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,64,0.017131733894348144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,128,0.016618667046229045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,32,0.017509333292643228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,10240,0.06651413440704346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,12288,0.07658987045288086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,16384,0.09467626412709554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,7168,0.05116159915924072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,8192,0.05606933434804281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,6144,0.046539731820424396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,16384,65536,0.40294720331827805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,5120,0.04180906613667806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,4096,0.03649066686630249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3584,0.03425813515981038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,3072,0.03390613396962484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1536,0.023002666234970093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2560,0.028562132517496747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,2048,0.02544426719347636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,1024,0.02065066695213318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,768,0.019769599040349327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,512,0.01741973360379537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,256,0.016549332936604818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,128,0.016156799594561257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,64,0.016360533237457276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,32,0.016614400347073875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,12288,65536,0.33954881032307943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,12288,0.06909226576487223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,10240,0.06053013404210409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,16384,0.08570773601531982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,7168,0.046666665871938066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,6144,0.042610132694244386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,5120,0.03805760145187378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,8192,0.055301332473754884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3072,0.028756266832351683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2560,0.026209066311518352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,4096,0.03359146515528361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,3584,0.03162239988644917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,2048,0.024146133661270143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1536,0.02203306754430135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,1024,0.020097066958745323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,768,0.01888426740964254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,512,0.017499732971191406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,256,0.01662826637427012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,128,0.016152532895406087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,64,0.016300800442695617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,32,0.016379732886950174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,10240,65536,0.3069525400797526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,10240,0.050760531425476076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,12288,0.058371198177337644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,16384,0.07112106482187906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,8192,0.047416532039642335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,7168,0.04028160174687703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,6144,0.0369045337041219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,5120,0.0332202672958374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3584,0.027005867163340254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,4096,0.029623466730117797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,3072,0.025756800174713136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2560,0.023431466023127238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,2048,0.022048000494639078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1024,0.01886506676673889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,1536,0.02065599958101908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,768,0.01672533353169759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,512,0.01636373301347097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,256,0.015938133001327515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,64,0.01601066688696543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,128,0.015548800428708395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,32,0.016270933548609416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,8192,65536,0.23994240760803223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,12288,0.05437973340352377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,10240,0.047465598583221434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,16384,0.06554773251215616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,8192,0.041157333056132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,6144,0.034696535269419355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,7168,0.03800106843312581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,5120,0.03133226633071899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,4096,0.027509333690007527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3584,0.025779199600219727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2560,0.022819199164708457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,3072,0.02418559988339742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,2048,0.021338667472203574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1024,0.018003199497858682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,1536,0.02007466753323873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,768,0.016551466782887776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,512,0.016288000345230102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,256,0.015958399573961893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,128,0.015989333391189575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,64,0.01588586668173472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,32,0.016110933820406594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,7168,65536,0.22801599502563477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,10240,0.04432426691055298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,12288,0.05027519861857096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,16384,0.06068053245544434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,8192,0.03793600002924601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,5120,0.02957013249397278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,4096,0.026029866933822633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,7168,0.03560320138931274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,6144,0.03249386747678121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3584,0.024565333127975465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,3072,0.023065600792566934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2048,0.020593067010243736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,2560,0.022374399503072104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1536,0.019695999224980672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,512,0.016378666957219443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,1024,0.01689066688219706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,768,0.01665279964605967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,256,0.015845333536465965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,128,0.01551040013631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,32,0.015808000167210897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,64,0.01567466656366984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,6144,65536,0.20071892738342284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,8192,0.03569600184758504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,10240,0.04142080148061116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,12288,0.04653653303782145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,16384,0.0566271980603536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,5120,0.027309866746266682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,4096,0.0248416006565094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,6144,0.030033065875371294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,7168,0.03344000180562337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3072,0.022674133380254112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,3584,0.023749333620071412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2048,0.020568533738454183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,2560,0.021885865926742555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1536,0.01924906571706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,1024,0.01734293301900228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,768,0.017084799210230508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,512,0.016492799917856852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,65536,0.1759552001953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,256,0.015852800011634825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,64,0.015458133816719056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,128,0.015598932902018229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,5120,32,0.015542399883270264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,8192,0.03131626645723979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,10240,0.03758506774902344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,12288,0.04779946804046631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,16384,0.05522880156834921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,7168,0.028812799851099653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,6144,0.02686079939206441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,5120,0.02440426747004191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,4096,0.022529067595799764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3584,0.022053333123524983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2560,0.019898666938145956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,2048,0.018902399142583213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,3072,0.021001599232355752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,768,0.016434133052825928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1536,0.017467733224232992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,1024,0.016025599837303162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,512,0.015824000040690102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,65536,0.1513216018676758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,256,0.015638400117556253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,64,0.015436800320943198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,128,0.015170133113861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,4096,32,0.015571199854214988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,8192,0.02895786762237549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,12288,0.0382698655128479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,10240,0.034134399890899655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,16384,0.04625279903411865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,7168,0.027007999022801717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,6144,0.025217066208521526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,5120,0.02348053256670634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,4096,0.021884800990422566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3584,0.020811732610066733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,3072,0.02033920089403788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2560,0.019444266955057778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,65536,0.13941973050435383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,2048,0.017767467101415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1536,0.01667733391125997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,1024,0.01632426679134369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,768,0.01586666703224182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,512,0.015820800264676412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,256,0.01529813309510549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,128,0.01537493367989858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,64,0.015432533621788026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3584,32,0.01532906691233317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,8192,0.030842665831247968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,10240,0.036817065874735516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,12288,0.04062933524449666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,16384,0.04874453147252401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,6144,0.024412800868352257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,7168,0.02598186731338501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,5120,0.022899200518925987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,4096,0.02143893241882324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3584,0.020777599016825358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,3072,0.020105600357055664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2560,0.01854506731033325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,2048,0.01773653427759806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,65536,0.1304522673288981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1536,0.017033600807189943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,1024,0.016219733158747356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,768,0.016139733791351318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,512,0.0156960000594457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,256,0.015338666240374246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,128,0.015260799725850423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,64,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,3072,32,0.015369600057601929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,12288,0.03775146802266439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,10240,0.033548800150553386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,16384,0.045501867930094406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,8192,0.02880000074704488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,7168,0.027369600534439088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,5120,0.022516266504923502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,6144,0.025542400280634564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,4096,0.020977065960566203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3584,0.020068265994389854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,3072,0.01979093352953593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2048,0.017036799589792886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,2560,0.0182805339495341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,65536,0.11579840183258057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1536,0.016821332772572837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,768,0.015801599621772765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,1024,0.016353066762288412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,256,0.015371732910474143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,512,0.01574186682701111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,128,0.015082666277885437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,32,0.015290666619936624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2560,64,0.015131733814875283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,12288,0.0359007994333903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,16384,0.043755733966827394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,10240,0.03151253263155619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,8192,0.027843199173609418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,7168,0.026503467559814455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,6144,0.025600000222524004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,5120,0.021974400679270426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,4096,0.020592000087102255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3584,0.01990506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,65536,0.10391146341959637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2560,0.018207999070485432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,3072,0.019309866428375243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1536,0.016622933745384216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,2048,0.017554134130477905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,1024,0.016023466984430947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,768,0.015800533692042033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,512,0.01530026694138845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,256,0.01524906655152639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,128,0.01493333379427592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,64,0.015110400319099427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,2048,32,0.015367466211318969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,16384,0.04101226727167766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,12288,0.03337706724802653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,10240,0.030583467086156207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,8192,0.02707093358039856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,7168,0.0253440002600352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,6144,0.02397226691246033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,5120,0.02116159995396932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,4096,0.019016534090042114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3584,0.01882666746775309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,65536,0.09234560330708821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2560,0.017619200547536216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,2048,0.017090133825937905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,3072,0.01776960094769796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1536,0.01664959987004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,768,0.015962666273117064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,512,0.015602133671442666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,1024,0.01590933303038279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,256,0.015191466609636942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,128,0.014985600113868713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,64,0.01504853367805481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1536,32,0.01513813336690267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,16384,0.03363413413365682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,12288,0.028678399324417115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,8192,0.02421440084775289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,10240,0.026447999477386474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,7168,0.02299626668294271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,6144,0.02182506720225016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,65536,0.08051946957906088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,5120,0.020819199085235596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,4096,0.020333866278330483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3584,0.01871466636657715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,3072,0.018257067600886027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2048,0.017145599921544394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,2560,0.017473065853118898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1536,0.016532267133394875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,1024,0.01590720017751058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,768,0.015853866934776306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,512,0.015575466553370157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,256,0.015160533785820007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,128,0.014912000298500061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,64,0.014833066860834757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,1024,32,0.014973866939544677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,12288,0.028051199515660603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,16384,0.03215786616007487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,8192,0.023546665906906128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,10240,0.025949867566426595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,65536,0.07533973058064779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,7168,0.02174293398857117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,6144,0.021333332856496176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,5120,0.01948053240776062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,4096,0.01877973278363546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3072,0.0180351992448171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,3584,0.018343466520309448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2560,0.01748266617457072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,2048,0.017127466201782227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1536,0.016284799575805663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,1024,0.015887999534606935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,768,0.01591146687666575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,256,0.015026133259137472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,512,0.015332266688346863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,128,0.014814933141072592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,64,0.014845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,768,32,0.014915200074513755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,12288,0.026741333802541095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,16384,0.030076799790064494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,8192,0.022809600830078124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,10240,0.02507413427035014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,7168,0.02188160022099813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,65536,0.07426666418711345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,6144,0.022219733397165934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,5120,0.019637332359949747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,4096,0.019056000312169395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3584,0.018177066246668497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,3072,0.017779199282328288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2048,0.016729599237442015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,2560,0.017101866006851197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1536,0.01651413341363271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,1024,0.01576640009880066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,768,0.015833600362141927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,512,0.015520000457763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,256,0.01495253344376882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,128,0.014939733346303306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,64,0.014807466665903726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,512,32,0.014909866452217101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,16384,0.025450666745503742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,10240,0.02290133237838745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,12288,0.023502933979034423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,8192,0.02177493373552958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,65536,0.06593600114186605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,7168,0.02130026618639628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,4096,0.0187882661819458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,6144,0.020794665813446044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3584,0.017997866868972777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,5120,0.01944533387819926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2560,0.017206400632858276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1536,0.01632213294506073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,3072,0.01759786605834961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,2048,0.016587733229001363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,1024,0.015677866339683533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,768,0.01564586659272512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,512,0.015266133348147073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,256,0.014846932888031007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,128,0.014915200074513755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,64,0.01469013293584188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,256,32,0.01480959951877594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,16384,0.024856533606847128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,12288,0.023131734132766722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,10240,0.022592000166575112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,65536,0.058290131886800135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,8192,0.021447465817133585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,7168,0.021466666460037233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,6144,0.020639999707539877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,5120,0.019375999768575035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,4096,0.018756266434987387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3584,0.018067200978597008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2560,0.017146666844685875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,3072,0.01770026683807373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,2048,0.016760534048080443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1536,0.016408532857894897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,1024,0.01567039986451467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,768,0.015643733739852905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,512,0.015159466862678527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,256,0.014946132898330688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,128,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,64,0.014775466918945313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,128,32,0.014643200238545737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,16384,0.024970666567484538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,12288,0.022985599438349404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,10240,0.022693334023157756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,65536,0.05363946755727132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,8192,0.02127466599146525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,7168,0.021444267034530638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,6144,0.02078933318456014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,5120,0.019409066438674925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3584,0.018114133675893148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,4096,0.018489599227905273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,3072,0.017568000157674155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2560,0.01696853240331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,2048,0.016797866423924765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,768,0.015595733126004537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,256,0.015000533064206442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,512,0.015188266833623251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1536,0.01609173317750295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,1024,0.015709867080052696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,64,0.014778666694959006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,128,0.014892799655596414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,64,32,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,16384,0.024829866488774617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,12288,0.02305493354797363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,65536,0.050546133518218996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,10240,0.02254613240559896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,8192,0.021498666206995646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,7168,0.021218132972717286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,6144,0.02039466698964437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,4096,0.01859626571337382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,5120,0.019512534141540527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3584,0.01799466609954834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,3072,0.017603200674057008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2560,0.017262933651606242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,2048,0.016822399695714314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1536,0.01628373364607493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,1024,0.015603199601173401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,768,0.0155349334081014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,512,0.015307733416557312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,256,0.01495573321978251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,128,0.014826666315396628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,64,0.0147189329067866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,160,32,32,0.014591999848683677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,7168,0.18448959986368815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,8192,0.2096778710683187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,10240,0.25438613891601564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,12288,0.3018730799357096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,5120,0.13889066378275555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,6144,0.16161386171976727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,4096,0.11665919621785481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,16384,0.39354238510131834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3584,0.10421546300252278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,3072,0.09303146998087565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2048,0.07002240022023519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,2560,0.0809653361638387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1536,0.06230719884236654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,1024,0.045373864968617755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,128,0.02189226746559143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,256,0.024692267179489136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,64,0.021635200579961142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,768,0.039255468050638835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,512,0.031897600491841635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,65536,32,0.021317332983016968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,8192,0.06637760003407797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,10240,0.07810986836751302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,12288,0.09026559988657633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,16384,0.11341439882914225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,7168,0.06009920040766398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,6144,0.05435306628545126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,5120,0.04824426571528117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,4096,0.04189759890238444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2560,0.03292693297068278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3584,0.03904533386230469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,3072,0.03573013146718343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,2048,0.02872106631596883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1536,0.02491626739501953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,1024,0.021780266364415487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,768,0.02027946710586548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,512,0.018989866971969603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,256,0.016551466782887776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,128,0.01646719972292582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,64,0.01686720053354899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,32,0.016825600465138753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,10240,0.06324586470921835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,12288,0.07299839655558268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,16384,0.09033173720041911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,8192,0.054324265321095785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,6144,0.045032533009847005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,7168,0.04970879952112834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,16384,65536,0.39526294072469076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,5120,0.0402730663617452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3584,0.033429332574208576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,4096,0.03523733218510945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,3072,0.03311040004094441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2560,0.027811199426651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1536,0.022219733397165934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,2048,0.024727465709050496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,1024,0.020111999909083047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,512,0.017061332861582436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,768,0.019287467002868652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,256,0.016203733285268147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,128,0.01588053305943807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,64,0.016302933295567833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,32,0.01639359990755717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,12288,65536,0.3273280143737793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,10240,0.056170666217803956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,12288,0.06481706698735555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,16384,0.07966612974802653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,7168,0.044438401858011886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,8192,0.053053867816925046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,6144,0.040573867162068684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,5120,0.03655466636021932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3072,0.027560534079869588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2560,0.02542080084482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,3584,0.030001066128412884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,4096,0.03224213322003682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,2048,0.023149865865707397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1024,0.019346133867899577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,768,0.018042665719985963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,1536,0.021277866760889688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,512,0.017042134205500284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,256,0.016055466731389363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,128,0.015715199708938598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,64,0.016200533509254454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,32,0.016150400042533875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,10240,65536,0.29194987614949547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,10240,0.04886399904886882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,12288,0.056195199489593506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,16384,0.06807146867116293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,8192,0.04640853404998779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,7168,0.03906346559524536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,6144,0.03597226540247599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,5120,0.0321727991104126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3584,0.026579199234644572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,4096,0.028699733813603717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,3072,0.024682666858037314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2560,0.023011199633280435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,2048,0.021571199099222817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1536,0.02003093361854553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,1024,0.018175999323527016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,768,0.016425599654515587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,512,0.016323199868202208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,256,0.015784533818562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,64,0.01572266618410746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,128,0.01528320014476776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,32,0.015875200430552162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,8192,65536,0.23373333613077799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,10240,0.0458026647567749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,12288,0.052245334784189856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,16384,0.06320106585820516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,8192,0.03985600074132283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,4096,0.026553599039713542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,7168,0.03612373272577922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,6144,0.03321066697438558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,5120,0.0302293340365092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3584,0.02505279978116353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,3072,0.02358506719271342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2048,0.0209279994169871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,2560,0.02206826607386271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1536,0.019514666001001994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,1024,0.017483733097712197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,768,0.016707199811935424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,512,0.015918933351834617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,256,0.015599999825159708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,128,0.015435733397801719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,64,0.015614933768908181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,32,0.015424000223477683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,7168,65536,0.21023252805074058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,16384,0.05728213389714559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,10240,0.041698133945465087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,12288,0.04717973470687866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,8192,0.03644479910532634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,4096,0.024769065777460735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,5120,0.02762133280436198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,7168,0.033164799213409424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,6144,0.030830933650334673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3584,0.023476266860961915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,3072,0.022116265694300332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2560,0.021282132466634116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,2048,0.020154666900634766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1536,0.01881813406944275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,768,0.01625599960486094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,1024,0.0161461333433787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,512,0.01586560010910034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,256,0.015321600437164306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,128,0.0151146670182546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,65536,0.20357119242350258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,64,0.015502933661142984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,6144,32,0.015585066874821982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,12288,0.04325226545333862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,8192,0.0335317333539327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,10240,0.03768426577250163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,16384,0.05221226612726847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,4096,0.023089067141215006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,5120,0.0252895991007487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,6144,0.027935999631881713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,7168,0.030690133571624756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3584,0.022204800446828207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,3072,0.021218132972717286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2560,0.019953066110610963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,2048,0.019371734062830607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1536,0.01794559955596924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,768,0.01593280037244161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,1024,0.016008533040682473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,512,0.01575573285420736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,256,0.015523200233777365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,65536,0.16719147364298503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,128,0.015042133132616677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,64,0.015202132860819497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,5120,32,0.015256533026695251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,8192,0.029948800802230835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,10240,0.034295467535654704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,16384,0.046256001790364584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,12288,0.03819626569747925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,7168,0.027309866746266682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,6144,0.02587946653366089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,4096,0.021976532538731892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,5120,0.023507199684778848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3584,0.021208532651265464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,3072,0.020436267058054604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2560,0.019831466674804687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,2048,0.0187008003393809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1536,0.016489600141843162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,65536,0.1431829293568929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,1024,0.016109866897265117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,768,0.015808000167210897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,512,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,256,0.01530026694138845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,128,0.01508799990018209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,64,0.015541332960128783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,4096,32,0.015494400262832641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,16384,0.04298773209253947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,8192,0.02805226643880208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,12288,0.036269867420196535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,10240,0.03260266582171122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,7168,0.026309333244959515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,5120,0.023270400365193684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,6144,0.024653865893681844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,4096,0.021502933899561563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3584,0.021146667003631592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,3072,0.020297600825627645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2560,0.019425066312154134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,2048,0.01795733372370402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,65536,0.13213760058085125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1024,0.01602026621500651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,1536,0.016736000776290894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,768,0.015929599603017174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,256,0.015546666582425437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,128,0.015245866775512696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,512,0.01581333378950755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,64,0.015411200126012168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3584,32,0.015213867028554281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,10240,0.030158933003743487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,8192,0.026723200082778932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,12288,0.03439146677652995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,16384,0.04102186759312947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,6144,0.023536000649134317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,7168,0.02490133245786031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,5120,0.02251733342806498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,4096,0.02085439960161845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3584,0.020431999365488687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,3072,0.019796266158421835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2560,0.018795732657114664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,2048,0.017292799552281697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1536,0.01647040049235026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,1024,0.016189866264661155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,65536,0.1192245324452718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,768,0.015936000148455302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,512,0.015842133760452272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,256,0.015161599715550741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,128,0.015130666891733804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,64,0.01516800026098887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,3072,32,0.015000533064206442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,12288,0.031795199712117514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,10240,0.028384000062942505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,8192,0.02550613284111023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,16384,0.0386186679204305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,7168,0.0240447998046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,6144,0.02289066712061564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,5120,0.02175146738688151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,4096,0.020702934265136717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3584,0.019977599382400513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,3072,0.018883200486501057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,65536,0.1078495979309082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2560,0.017452800273895265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,2048,0.017145599921544394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1536,0.01667520006497701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,1024,0.0160480002562205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,768,0.015961600343386333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,512,0.0156960000594457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,128,0.015389866630236306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,256,0.015255467096964518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,64,0.014993066589037577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2560,32,0.015083733201026916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,12288,0.031598933537801105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,10240,0.029322665929794312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,16384,0.03725333213806152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,8192,0.025224532683690386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,6144,0.023062400023142495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,7168,0.024337067206700643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,5120,0.02141439914703369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,4096,0.019781333208084107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3584,0.019483733177185058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,65536,0.09666666984558106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,3072,0.017655466000239053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2560,0.017744000752766928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,2048,0.017033600807189943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1536,0.016434133052825928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,1024,0.01592639982700348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,512,0.015517866611480713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,768,0.01560426652431488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,256,0.015046399831771851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,128,0.01479573349157969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,64,0.01495039959748586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,2048,32,0.015060266852378846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,16384,0.0333898663520813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,12288,0.027756800254185993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,10240,0.025568000475565594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,8192,0.023441066344579063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,7168,0.022370133797327676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,6144,0.02148800094922384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,5120,0.020467199881871543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,65536,0.08353919982910156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,4096,0.018973867098490395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3584,0.018231467405954997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,3072,0.017512534062067667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2560,0.01718399922053019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1536,0.016311466693878174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,2048,0.016757333278656007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,1024,0.015875200430552162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,256,0.01508799990018209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,512,0.01546346644560496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,768,0.015709867080052696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,128,0.014822399616241455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,64,0.014923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1536,32,0.014907733599344889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,12288,0.026122667392094928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,16384,0.02886506716410319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,10240,0.02540480097134908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,8192,0.023758933941523234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,5120,0.01965973377227783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,7168,0.022025599082310995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,65536,0.07242560386657715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,6144,0.02111466725667318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,4096,0.018652800718943277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3584,0.018194133043289186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2560,0.01733013391494751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,3072,0.017661867539087932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1536,0.01613866686820984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,2048,0.016709333658218382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,1024,0.015727999806404113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,256,0.014987732966740927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,768,0.015577600399653117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,512,0.015265066425005594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,128,0.01474240024884542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,64,0.014871467153231302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,1024,32,0.014862933754920959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,12288,0.02513706684112549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,16384,0.02799466649691264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,10240,0.024053333202997844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,8192,0.02198293407758077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,7168,0.021760000785191854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,65536,0.0664693315823873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,5120,0.01945706605911255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,6144,0.02013546625773112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,4096,0.01853013237317403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3584,0.01797013282775879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2560,0.01722453236579895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,3072,0.017618133624394735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1536,0.016212266683578492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,2048,0.01663146714369456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,1024,0.01572266618410746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,768,0.015607466300328573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,256,0.015064533551534018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,512,0.015372799833615622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,128,0.014786133170127868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,64,0.014780799547831217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,768,32,0.014794666568438211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,16384,0.02632853388786316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,12288,0.02542080084482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,10240,0.024099200963974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,8192,0.021683200200398763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,65536,0.06043306589126587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,7168,0.021167999505996703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,5120,0.01943040092786153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,6144,0.020422399044036865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,4096,0.018454400698343913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3584,0.01798186699549357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2560,0.017148800690968833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1536,0.016210132837295534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,3072,0.017595734198888144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,2048,0.016810667514801026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,1024,0.015737600127855935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,768,0.01572266618410746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,512,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,256,0.014959999918937683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,128,0.014839466412862143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,64,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,512,32,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,12288,0.02197973330815633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,16384,0.024685867627461753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,10240,0.021913599967956544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,65536,0.054622932275136316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,8192,0.021010132630666097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,7168,0.020779732863108316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,6144,0.020447999238967896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,5120,0.019291732708613077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3584,0.018023467063903807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,4096,0.018362667163213095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,3072,0.0172160009543101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2560,0.017062399784723917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,2048,0.016591999928156534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1536,0.015904000401496886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,1024,0.015701333681742348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,768,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,512,0.015219199657440185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,256,0.014961066842079162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,128,0.014686933159828186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,64,0.014776532848676046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,256,32,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,16384,0.02365760008494059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,12288,0.022180267175038657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,10240,0.02172799905141195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,65536,0.046528001626332596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,8192,0.02071253259976705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,7168,0.020939733584721884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,6144,0.020057600736618043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,5120,0.019377066691716512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3072,0.017384533087412515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,3584,0.017942400773366292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,4096,0.018309332927068076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2560,0.016857600212097167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,2048,0.016660267114639284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1536,0.016083199779192606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,1024,0.015737600127855935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,768,0.01541866660118103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,512,0.015094400445620219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,128,0.014797866344451904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,256,0.01493333379427592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,64,0.014738133549690247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,128,32,0.01463573376337687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,16384,0.023846399784088135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,12288,0.021959465742111207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,65536,0.0433898647626241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,10240,0.0218560000260671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,8192,0.020716800292332967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,7168,0.02068480054537455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,6144,0.02020053267478943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,5120,0.01927893360455831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,4096,0.018236800034840902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3584,0.01813439925511678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,3072,0.017566933234532674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2560,0.017061332861582436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,2048,0.016523733735084534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1536,0.015980799992879234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,1024,0.01571626663208008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,768,0.015520000457763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,512,0.01511679987112681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,256,0.014948266744613647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,128,0.014707199732462563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,64,0.014791466792424521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,64,32,0.014696533481280008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,16384,0.02376746733983358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,65536,0.042181332906087235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,8192,0.02045546571413676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,10240,0.02173973321914673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,12288,0.02178986668586731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,5120,0.01909439961115519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,6144,0.020070399840672812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,7168,0.020809600750605263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,4096,0.018367999792099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3584,0.017838933070500693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,3072,0.017448532581329345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2560,0.017053866386413576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,2048,0.0166293332974116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1024,0.01565120021502177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,1536,0.016053332885106405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,768,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,512,0.015200000007947287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,128,0.014685866236686707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,256,0.014818132917086283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,64,0.014622933665911355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,128,32,32,0.014625066518783569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,7168,0.18021440505981445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,8192,0.20382827123006186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,10240,0.2496469338734945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,12288,0.2948394775390625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,6144,0.15758934020996093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,5120,0.1350165367126465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,4096,0.11243626276652019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,16384,0.38595625559488933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3584,0.10119466781616211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2560,0.0781002680460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1536,0.05981546640396118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,2048,0.06677652994791666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,3072,0.0898517370223999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,1024,0.0431167999903361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,256,0.02331733306248983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,128,0.020819199085235596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,64,0.019036799669265747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,512,0.030104533831278486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,768,0.03726613521575928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,65536,32,0.019576533635457357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,8192,0.06429440180460612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,10240,0.07528106371561685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,12288,0.08727467060089111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,16384,0.10961493651072185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,7168,0.05839999914169312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,6144,0.052590934435526523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,5120,0.046803200244903566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,4096,0.040515200297037764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2560,0.03161279956499736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3072,0.03473600149154663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,3584,0.03771946827570598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,2048,0.02808319926261902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1536,0.024309333165486655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,1024,0.021305600802103676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,768,0.020057600736618043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,512,0.018654932578404747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,256,0.01615466674168905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,128,0.015873066584269204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,64,0.016313599546750386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,32,0.01624853312969208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,10240,0.06092693408330282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,12288,0.07207573254903157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,16384,0.08907840251922608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,8192,0.05270293156305948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,6144,0.045339731375376384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,7168,0.04883946577707927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,16384,65536,0.3821962674458822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,4096,0.035846400260925296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3584,0.03292693297068278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,5120,0.03970133463541667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,3072,0.03313279946645101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2560,0.027833600838979084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,2048,0.025018666187922162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1024,0.020520534118016562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,1536,0.022284799814224245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,768,0.01920959949493408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,256,0.01602666676044464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,512,0.01672640045483907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,128,0.015549866358439126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,32,0.01602666676044464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,64,0.015896532932917276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,12288,65536,0.3133013407389323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,10240,0.05472533305486044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,12288,0.0630133350690206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,16384,0.07690239747365316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,8192,0.051856001218159996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,6144,0.03975679874420166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,7168,0.04374399979909261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,5120,0.0356768012046814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,4096,0.03167999982833862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3072,0.027155200640360515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2560,0.025158399343490602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,3584,0.02941333254178365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1536,0.021178666750590006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,2048,0.023220266898473105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,1024,0.019393066565195717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,768,0.018281600872675576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,512,0.01680319905281067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,256,0.01600320041179657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,128,0.015476266543070475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,64,0.01572266618410746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,32,0.015570132931073507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,10240,65536,0.27894185384114584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,10240,0.047220265865325926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,12288,0.05414719978968302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,16384,0.06565226713816324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,8192,0.044354132811228436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,5120,0.031666133801142374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,6144,0.0348469336827596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,7168,0.037835733095804853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3584,0.02615999976793925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,3072,0.02421440084775289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2560,0.022782933712005616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,4096,0.027755733331044512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1536,0.019734400510787963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,2048,0.02131839990615845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,768,0.016075733304023742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,1024,0.018205867211023966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,512,0.016334933042526246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,256,0.01570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,128,0.015315199891726175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,64,0.015870933731396995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,32,0.015752533078193666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,8192,65536,0.22197225888570152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,8192,0.03858666817347209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,12288,0.049930667877197264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,10240,0.04382933378219604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,16384,0.06017066637674967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,4096,0.025940267244974773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,5120,0.029500800371170043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,7168,0.03532906770706177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,6144,0.03271680076917012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3584,0.02453440030415853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2560,0.022769065697987874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,3072,0.023468800385793052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,2048,0.020729599396387736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1536,0.019211733341217042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,1024,0.016810667514801026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,768,0.01622293293476105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,512,0.015963733196258545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,128,0.015539200107256571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,256,0.015452800194422403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,64,0.015610667069753012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,32,0.015470932920773825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,7168,65536,0.20928319295247397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,12288,0.04573226769765218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,8192,0.03548693259557088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,10240,0.04050346612930298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,16384,0.054654932022094725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,4096,0.024784000714619954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,5120,0.027031467358271284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,7168,0.03282026648521423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,6144,0.030166399478912354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3072,0.022362667322158813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,3584,0.023516800006230673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2560,0.021305600802103676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,2048,0.01994133392969767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1536,0.018971733252207437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,768,0.01615466674168905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,512,0.01585493286450704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,1024,0.01665493349234263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,128,0.01532266636689504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,256,0.015537066260973611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,64,0.015384533007939658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,32,0.015359999736150107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,6144,65536,0.18171520233154298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,8192,0.03273599942525228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,12288,0.04133653243382772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,10240,0.03674240112304687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,16384,0.04970133304595947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,6144,0.0274944007396698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,5120,0.02502080003420512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,7168,0.030061866839726763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,4096,0.02322346568107605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3584,0.022437334060668945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,3072,0.021150932709376017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2560,0.020407466093699138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,2048,0.01917546590169271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1536,0.01758613387743632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,1024,0.01588053305943807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,768,0.015966932972272238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,512,0.015899733702341715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,256,0.015269333124160766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,65536,0.1574325402577718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,128,0.015075199802716575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,64,0.015352533260981242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,5120,32,0.01529706617196401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,10240,0.03291626572608948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,12288,0.04157439867655437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,16384,0.04846506516138713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,8192,0.028895999987920123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,6144,0.025214932362238568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,5120,0.023373866081237794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,7168,0.02699199914932251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,4096,0.021754666169484457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3584,0.02118399937947591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,3072,0.020374399423599244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2560,0.01951040029525757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,2048,0.018423465887705485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1536,0.016170666615168253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,1024,0.016330666343371072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,768,0.01590506633122762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,65536,0.1346175988515218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,512,0.015577600399653117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,256,0.01516800026098887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,128,0.015177599589029946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,64,0.01541866660118103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,4096,32,0.015404799580574035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,8192,0.026987733443578084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,10240,0.030882134040196733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,12288,0.03592746655146281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,16384,0.04559146563212077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,7168,0.02546773354212443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,6144,0.023987199862798056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,5120,0.022725333770116173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,4096,0.0213919997215271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3584,0.020520534118016562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,3072,0.02005013426144918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2560,0.01910933256149292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,2048,0.017529600858688356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,65536,0.12380373477935791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1536,0.016105600198109946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,1024,0.016038399934768677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,768,0.015966932972272238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,512,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,256,0.01527466674645742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,32,0.015080533425013223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,128,0.015034666657447815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3584,64,0.01520746648311615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,8192,0.025674666961034136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,10240,0.02911786635716756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,16384,0.042738131682078045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,12288,0.03247146606445313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,7168,0.02441493272781372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,6144,0.023139200607935586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,4096,0.02057173252105713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,5120,0.022036266326904298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3584,0.02002453406651815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2560,0.018396800756454466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,3072,0.01946880022684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,65536,0.11337172985076904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,2048,0.01737920045852661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1536,0.016658133268356322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,1024,0.016012799739837647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,768,0.01557866632938385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,512,0.015464533368746439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,256,0.015204266707102457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,128,0.01504746675491333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,64,0.015092266599337259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,3072,32,0.014922666549682616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,12288,0.03231253425280253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,10240,0.02855679988861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,16384,0.038680533568064376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,8192,0.025788799921671553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,7168,0.024386133750279745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,6144,0.023202133178710938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,5120,0.021410133441289267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,4096,0.01994880040486654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3584,0.019766400257746376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,3072,0.01821546753247579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,65536,0.10193493366241455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2560,0.017271467049916587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,2048,0.017030400037765504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1536,0.016017066438992818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,1024,0.016108799974123636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,768,0.015546666582425437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,128,0.015030399958292643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,512,0.015629866719245912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,256,0.015254400173823037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,64,0.015155200163523355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2560,32,0.014917332927385965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,12288,0.0286463995774587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,10240,0.026225066184997557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,8192,0.023516800006230673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,16384,0.035241599877675375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,6144,0.022399999698003135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,7168,0.022923733790715536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,5120,0.020733867088953653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,4096,0.019641600052515664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3584,0.019038933515548705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,65536,0.08685973485310873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2560,0.017594667275746663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,3072,0.01857066750526428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,2048,0.016809600591659545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1536,0.016555733482042947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,1024,0.015961600343386333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,768,0.01580693324406942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,512,0.015508266290028891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,256,0.015107199549674988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,128,0.014870400230089823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,64,0.015096533298492431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,2048,32,0.014979199568430582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,16384,0.03060800035794576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,12288,0.026743467648824053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,10240,0.02467199961344401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,8192,0.02296746571858724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,7168,0.022003199656804404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,6144,0.02127679983774821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,5120,0.01978666583697001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,4096,0.018231467405954997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,65536,0.07652906576792398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3584,0.018260266383488974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,3072,0.01799466609954834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2560,0.01718826691309611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,2048,0.01679146687189738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1536,0.01628373364607493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,1024,0.015892266233762106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,768,0.015638400117556253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,512,0.015481600165367126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,256,0.015200000007947287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,128,0.01486186683177948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,64,0.014884266257286071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1536,32,0.014885333180427552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,16384,0.02839360038439433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,10240,0.024581333001454674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,8192,0.02323413292566935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,12288,0.02576853235562642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,65536,0.06434666713078817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,5120,0.019691733519236247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,6144,0.020895999670028687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,7168,0.021709867318471274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,4096,0.018179200092951455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3072,0.017451733350753784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,3584,0.0178272008895874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2560,0.017025067408879598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,2048,0.016521599888801575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1536,0.015923200050989787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,1024,0.01583573321501414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,768,0.01564800043900808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,512,0.015280000368754067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,256,0.014944000045458474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,128,0.014666666587193808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,64,0.014949333667755128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,1024,32,0.014763733744621277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,16384,0.027349332968393963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,12288,0.02490453322728475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,10240,0.023628799120585124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,8192,0.021511467297871907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,65536,0.05896960099538168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,7168,0.021201066176096597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,6144,0.02037973403930664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,4096,0.01835626761118571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,5120,0.01941653291384379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3584,0.018127999703089395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,3072,0.01747200091679891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2560,0.01693333387374878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,2048,0.016476800044377647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1536,0.016114133596420287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,1024,0.015889066457748412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,512,0.015343999862670899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,768,0.015468800067901611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,256,0.015011200308799743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,128,0.014794666568438211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,64,0.014891733725865683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,768,32,0.014751999576886495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,16384,0.025148799022038777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,12288,0.022150399287541707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,8192,0.019899733861287437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,10240,0.0210207998752594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,65536,0.05373546679814657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,7168,0.02058773239453634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,6144,0.020248534282048543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,4096,0.018152532974878947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,5120,0.01911679903666178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3584,0.018170666694641114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,3072,0.017650133371353148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2560,0.01699413259824117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,2048,0.016583466529846193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1536,0.015990400314331056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,1024,0.015662933389345803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,512,0.015317333738009134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,768,0.015490133563677469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,256,0.014947199821472168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,128,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,64,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,512,32,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,65536,0.04500906864802043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,16384,0.0229695995648702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,8192,0.020243199666341145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,12288,0.021386667092641195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,10240,0.02093120018641154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,7168,0.020499199628829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,5120,0.019024000565210978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,6144,0.020202666521072388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,4096,0.018039466937383015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3584,0.017940266927083334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2560,0.01694933374722799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,3072,0.01741866668065389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,2048,0.016478932897249856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1536,0.01599253316720327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,1024,0.015784533818562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,768,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,512,0.0153546671072642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,256,0.014914133151372275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,128,0.014724266529083253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,64,0.014798933267593383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,256,32,0.014739200472831726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,65536,0.038097067674001055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,16384,0.022317866484324135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,8192,0.020297600825627645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,12288,0.02093440095583598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,10240,0.021005866924921672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,7168,0.02050559918085734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,5120,0.019382399320602418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,6144,0.02002453406651815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,4096,0.01799466609954834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3584,0.017718400557835898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,3072,0.017288533846537273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2560,0.01687893271446228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,2048,0.01646613379319509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1024,0.015525333086649575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,1536,0.015922133127848306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,768,0.015609600146611533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,512,0.015441067020098367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,256,0.01493119994799296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,128,0.01469013293584188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,64,0.014619732896486918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,128,32,0.014637866616249084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,16384,0.022712532679239908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,12288,0.021241599321365358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,65536,0.036433064937591554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,10240,0.020968532562255858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,8192,0.0200981338818868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,6144,0.01983039975166321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,7168,0.02071573336919149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,5120,0.01894186735153198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3584,0.017681066195170084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,4096,0.018183465798695883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,3072,0.017113600174585977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2560,0.01700906753540039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,2048,0.01634239951769511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1536,0.015957333644231162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,1024,0.015596800049146018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,768,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,512,0.015254400173823037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,256,0.014946132898330688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,128,0.014670933286348978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,64,0.014708266655604044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,16384,0.022012799978256226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,64,32,0.014548266927401224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,10240,0.020703999201456706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,12288,0.02092693249384562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,65536,0.0351146658261617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,8192,0.020140800873438516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,6144,0.019663999478022255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,7168,0.020105600357055664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,5120,0.019179733594258626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,4096,0.01790293256441752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3584,0.017666133244832356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,3072,0.01734506686528524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2048,0.016420267025629678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,2560,0.016900267203648886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1536,0.016035200158754984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,1024,0.015587199727694193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,768,0.015522133310635886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,512,0.015108266472816467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,128,0.01458346645037333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,256,0.014830933014551798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,64,0.014633599917093912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,96,32,32,0.014647466937700906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,7168,0.18628907203674316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,8192,0.2095253308614095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,10240,0.25582613945007326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,12288,0.30167465209960936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,6144,0.16192107200622557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,5120,0.13902506828308106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,4096,0.11362773577372234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,16384,0.3964650789896647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3584,0.10343786875406902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2560,0.08021972974141439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,2048,0.06739093462626139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,3072,0.0920245329538981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1024,0.04389546712239583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,1536,0.062386135260264076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,128,0.021005866924921672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,256,0.023324799537658692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,64,0.019128533204396565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,512,0.030742400884628297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,768,0.03899733225504558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,65536,32,0.01936533252398173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,10240,0.07529919942220052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,8192,0.06341546773910522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,12288,0.08705706596374511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,16384,0.10939626693725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,5120,0.046002133687337236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,7168,0.057475201288859044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,6144,0.052129065990448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,4096,0.04013546705245972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3072,0.03418879906336467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2560,0.0312991996606191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,2048,0.02763413389523824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,3584,0.037071998914082846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,768,0.01986453334490458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1536,0.02422399918238322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,512,0.018394666910171508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,1024,0.021373866001764934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,256,0.016195199886957803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,64,0.015982932845751443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,128,0.015871999661127727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,32,0.01614400049050649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,10240,0.06385279893875122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,12288,0.0738965352376302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,16384,0.09243733088175456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,8192,0.05454613367716471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,7168,0.04995413223902385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,6144,0.044870400428771974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,16384,65536,0.38013439178466796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,4096,0.035323735078175864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,5120,0.039961600303649904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3584,0.032646399736404416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,3072,0.031872000296910605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2560,0.027296000719070436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,2048,0.024971733490626015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1536,0.022807466983795165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,768,0.01918399930000305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,1024,0.019989333550135293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,512,0.016511999567349753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,256,0.015904000401496886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,128,0.01532906691233317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,32,0.015800533692042033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,64,0.015820800264676412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,10240,0.05450133482615153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,12288,0.06252906719843546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,12288,65536,0.30871785481770836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,16384,0.07633066972096761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,8192,0.04697386821111043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,6144,0.04251199960708618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,7168,0.04311360120773315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,5120,0.03540266752243042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2560,0.024872533480326333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3072,0.02733866572380066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,3584,0.02919466694196065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,4096,0.031506133079528806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,2048,0.02299306591351827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1536,0.021452800432840983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,1024,0.019588265816370645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,768,0.01806933283805847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,512,0.01627626617749532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,256,0.015833600362141927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,128,0.015385599931081137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,64,0.015711999932924905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,32,0.015631999572118124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,10240,65536,0.2720661481221517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,10240,0.04640640020370483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,16384,0.0639306664466858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,12288,0.052933335304260254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,8192,0.043866666158040364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,7168,0.03731199900309245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,6144,0.03423999945322673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,5120,0.03110719919204712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3584,0.025831466913223265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,4096,0.0280074675877889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,3072,0.023934932549794515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2560,0.022706133127212525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,2048,0.02093440095583598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1024,0.01818986733754476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,1536,0.019785600900650024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,768,0.01625279982884725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,256,0.015624533096949259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,512,0.01621333360671997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,128,0.015399466951688132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,64,0.015549866358439126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,32,0.01569706698258718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,8192,65536,0.21816746393839517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,12288,0.04863573312759399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,10240,0.042971734205881754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,16384,0.05874346494674683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,8192,0.03881173531214396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,4096,0.02595733404159546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,5120,0.02892799973487854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,6144,0.032034132877985635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,7168,0.0347488005956014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3584,0.024652800957361855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2560,0.021717333793640138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,3072,0.02309760053952535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,2048,0.020304000377655028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1536,0.01941546599070231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,1024,0.017218132813771568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,768,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,512,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,256,0.015415466825167336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,128,0.015478400389353433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,64,0.015410133202870688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,32,0.015479466319084168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,7168,65536,0.20978879928588867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,10240,0.039700265725453696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,12288,0.04456106821695964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,8192,0.03618239959081014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,16384,0.053407998879750576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,5120,0.0268394668896993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,4096,0.024232532580693564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,6144,0.029923200607299805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,7168,0.03225066661834717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3584,0.023177599906921385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,3072,0.021908267339070638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2560,0.02095359961191813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,2048,0.019721599419911702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1536,0.01914560000101725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,1024,0.01660160024960836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,768,0.016030933459599814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,512,0.01565120021502177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,256,0.015588266650835672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,128,0.015247999628384908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,65536,0.17754027048746746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,64,0.015179733435312906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,6144,32,0.015211733182271323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,12288,0.040775465965271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,16384,0.04807680050532023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,8192,0.03188479940096538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,10240,0.03590720097223918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,6144,0.027210666735966997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,5120,0.02474453250567118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,4096,0.023075199127197264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,7168,0.029549866914749146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3584,0.02199466625849406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2560,0.020165334145228066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,3072,0.02118826707204183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,2048,0.019099734226862588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1536,0.0178656001885732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,1024,0.016249600052833556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,512,0.015599999825159708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,768,0.015957333644231162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,65536,0.15326827367146809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,256,0.015264000495274863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,128,0.015191466609636942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,64,0.015127467115720114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,5120,32,0.015006933609644571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,16384,0.047261865933736165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,12288,0.03625066677729289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,8192,0.028333866596221925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,10240,0.03254080017407735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,7168,0.0264138658841451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,5120,0.02322453260421753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,6144,0.024875734249750772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,4096,0.02155733307202657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3584,0.02090453306833903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,3072,0.020028799772262573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2560,0.01913706660270691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,2048,0.018119466304779053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,65536,0.1306549310684204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1536,0.016131200393040977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,1024,0.015899733702341715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,768,0.01572160025437673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,512,0.015533866484959922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,256,0.015315199891726175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,128,0.01504853367805481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,64,0.015336533387502035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,4096,32,0.015315199891726175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,8192,0.026786132653554277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,10240,0.030418133735656737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,12288,0.03441280126571655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,16384,0.04410880009333293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,7168,0.025219200054804485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,6144,0.024001065889994302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,5120,0.02246826688448588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,4096,0.021115734179814657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3584,0.020441599686940513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,3072,0.01976213256518046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2560,0.018925867478052773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,2048,0.017516799767812095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,65536,0.1187872012456258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1024,0.01590293347835541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,768,0.015799466768900552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,1536,0.016504533092180886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,512,0.01546239952246348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,256,0.01530026694138845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,128,0.015004799763361613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,64,0.015149866541226705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3584,32,0.015034666657447815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,10240,0.031692800919214885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,8192,0.028621866305669146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,12288,0.03481173515319824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,16384,0.04078293244043986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,6144,0.023011199633280435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,7168,0.0240831991036733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,5120,0.021710934241612752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,4096,0.020517333348592123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3072,0.019222400585810342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,3584,0.01995733380317688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2560,0.01832853356997172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,65536,0.10806612968444824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,2048,0.017844265699386595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1536,0.016756266355514526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,1024,0.015941333770751954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,512,0.015501866738001505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,768,0.01577279965082804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,256,0.015148799618085226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,128,0.014990933736165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,64,0.01502293348312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,3072,32,0.0150026669104894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,12288,0.03200746575991313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,10240,0.02696746587753296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,8192,0.024116265773773193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,16384,0.03721706469853719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,7168,0.02308373252550761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,5120,0.020938666661580403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,6144,0.02204906741778056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,4096,0.020003199577331543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3584,0.019463467597961425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,65536,0.09690453211466471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,3072,0.01793280045191447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2560,0.01704853375752767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,2048,0.016530133287111917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1536,0.016316800316174825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,768,0.015761066476504007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,1024,0.016056533654530844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,256,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,512,0.015408000349998474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,128,0.014999467134475707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,64,0.015127467115720114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2560,32,0.014797866344451904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,12288,0.029231999317804975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,16384,0.03346773386001587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,10240,0.027401600281397504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,8192,0.023753599325815836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,7168,0.02249280015627543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,6144,0.021886932849884033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,5120,0.020693333943684895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,4096,0.01941759983698527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,65536,0.08237653573354085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3584,0.018143999576568603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,3072,0.017617066701253258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2560,0.017198934157689413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,2048,0.016670932372411094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1536,0.016275200247764587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,1024,0.01597653329372406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,768,0.015553067127863566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,512,0.015380266308784484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,256,0.015007999539375306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,128,0.014949333667755128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,32,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,2048,64,0.015016532937685647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,10240,0.024990934133529662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,8192,0.02357333302497864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,16384,0.029597866535186767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,12288,0.026331732670466106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,65536,0.07213013172149658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,7168,0.023192532857259116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,6144,0.022000000874201456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,5120,0.020078933238983153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,4096,0.01867626706759135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3584,0.018643200397491455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,3072,0.017832533518473307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2560,0.016909867525100708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,2048,0.016761600971221924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1536,0.016402133305867515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,1024,0.015864533185958863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,768,0.015717333555221556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,512,0.015490133563677469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,256,0.015054933230082192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,128,0.015007999539375306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,64,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1536,32,0.014962133765220643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,12288,0.025864533583323163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,16384,0.02675199906031291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,10240,0.023877332607905068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,8192,0.022445867458979286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,65536,0.06052480141321818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,5120,0.02029973268508911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,7168,0.02148053248723348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,6144,0.020642133553822835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,4096,0.018572799364725747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3584,0.01792959968249003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,3072,0.017515732844670614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2560,0.016953599452972413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,2048,0.016428800423940022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1536,0.016150400042533875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,1024,0.01589120030403137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,768,0.015502933661142984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,256,0.014923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,512,0.015254400173823037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,64,0.014897066354751586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,128,0.014773333072662353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,1024,32,0.014791466792424521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,16384,0.027062400182088213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,8192,0.020949333906173706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,12288,0.024727465709050496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,10240,0.022906666994094847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,65536,0.054593066374460854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,6144,0.019942400852839152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,5120,0.01898346741994222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,4096,0.017946666479110716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,7168,0.020402133464813232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2560,0.01679253379503886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,2048,0.016340266664822897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3072,0.01758613387743632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,3584,0.017641599973042807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1536,0.016152532895406087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,1024,0.015702399611473083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,256,0.014793599645296732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,768,0.015503999590873719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,512,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,128,0.014779733618100485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,64,0.014884266257286071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,768,32,0.01469546655813853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,16384,0.024752000967661537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,12288,0.023835732539494833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,10240,0.022613332668940226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,8192,0.020922666788101195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,65536,0.048368000984191896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,7168,0.02061333258946737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,6144,0.02027626633644104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,5120,0.018894932667414346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,4096,0.017977599302927652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3584,0.01749546726544698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,3072,0.017336533466974894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2560,0.016752000649770102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1536,0.016141866644223533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,2048,0.016361600160598753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,1024,0.01555519998073578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,768,0.015588266650835672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,256,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,512,0.015213867028554281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,128,0.014804266889890037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,64,0.014706133802731832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,512,32,0.014682666460673014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,12288,0.020758400360743202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,16384,0.02182719906171163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,8192,0.02000853419303894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,65536,0.039112532138824464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,10240,0.02096959948539734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,7168,0.020269866784413657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,6144,0.020055466890335084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,5120,0.018886399269104005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,4096,0.018023467063903807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3584,0.017602133750915527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,3072,0.01730453372001648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1536,0.016013866662979125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2560,0.016801067193349204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,2048,0.016314666469891867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,1024,0.01575040022532145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,768,0.01539306640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,512,0.015086932977040609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,128,0.014775466918945313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,256,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,32,0.014749866724014283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,256,64,0.014745600024859109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,16384,0.021517866849899293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,10240,0.02090880076090495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,12288,0.020529067516326903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,65536,0.03670719861984253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,8192,0.01990293264389038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,7168,0.020214400688807168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,6144,0.020013866821924846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,4096,0.0180842657883962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,5120,0.018675200144449868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3584,0.017590399583180746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,3072,0.01743146578470866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2560,0.016771199305852254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,2048,0.01623253325621287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1536,0.016131200393040977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,1024,0.015526400009791056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,768,0.015597866972287497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,512,0.015096533298492431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,256,0.014699733257293702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,128,0.014729600151379904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,64,0.014627200365066529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,128,32,0.014748799800872802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,16384,0.021334399779637657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,65536,0.034392531712849936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,10240,0.020415999492009482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,12288,0.020499199628829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,8192,0.019959467649459838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,7168,0.02012053330739339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,6144,0.019782400131225585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,5120,0.01858453353246053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,4096,0.017940266927083334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3584,0.01765226721763611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,3072,0.01691306630770365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2560,0.016753067572911583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,2048,0.016225066781044007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1536,0.015917866428693136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,1024,0.015560533603032431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,512,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,768,0.015380266308784484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,64,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,256,0.014871467153231302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,128,0.014689067006111145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,64,32,0.014682666460673014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,16384,0.02152106761932373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,12288,0.02029866576194763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,10240,0.02050666610399882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,65536,0.03516800006230672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,8192,0.019848533471425376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,7168,0.020167466004689535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,6144,0.019691733519236247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,5120,0.018566399812698364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,4096,0.017934934298197428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3584,0.017589332660039265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,3072,0.01718506614367167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2560,0.01665493349234263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,2048,0.01634773313999176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1536,0.015953066945075988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,1024,0.015522133310635886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,768,0.01532266636689504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,512,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,256,0.01490133305390676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,64,0.014637866616249084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,128,0.014631467064221701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,80,32,32,0.014594133694966635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,7168,0.18493653933207194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,8192,0.20522027015686034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,10240,0.2525695959726969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,12288,0.30011625289916993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,6144,0.160698668162028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,5120,0.13617599805196126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,4096,0.1111680030822754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,16384,0.39459520975748696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3584,0.10160640080769856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2560,0.07828266620635986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,2048,0.06499840021133423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,3072,0.09081386725107829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1024,0.042534399032592776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,1536,0.061109332243601475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,256,0.02253119945526123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,512,0.029762132962544756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,128,0.02012373407681783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,768,0.03742826779683431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,64,0.018845866123835243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,65536,32,0.018663465976715088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,8192,0.06507519880930582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,12288,0.08900159994761149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,10240,0.07640213171641032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,16384,0.11125333309173584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,5120,0.04753599961598714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,7168,0.05938453276952108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,6144,0.05313813289006551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,4096,0.04013653198877971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2560,0.030475733677546184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3072,0.03357119957605998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,3584,0.037718399365743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,2048,0.0270197331905365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1536,0.023727999130884806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,1024,0.02109653353691101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,768,0.019612799088160195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,512,0.017978666226069133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,256,0.015944533546765647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,128,0.01562879979610443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,64,0.01570026675860087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,32,0.015897599856058757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,10240,0.059646932284037266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,12288,0.06869973341623942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,16384,0.08494400183359782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,8192,0.051176532109578454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,7168,0.04669546683629354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,6144,0.0423253337542216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,16384,65536,0.4087509473164876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,4096,0.03339626789093018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,5120,0.0378602663675944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3584,0.03120959997177124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,3072,0.03101866642634074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1536,0.02151893377304077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2048,0.023731199900309245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,2560,0.026365866263707478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,1024,0.01944533387819926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,768,0.01845226685206095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,512,0.01574720044930776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,256,0.015686399737993875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,128,0.015460266669591268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,64,0.015468800067901611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,32,0.01567359964052836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,12288,65536,0.30180479685465494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,12288,0.060305066903432215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,10240,0.05265706777572632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,16384,0.07376853624979654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,8192,0.049711998303731283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,6144,0.037994666894276934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,5120,0.034209068616231283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,7168,0.04152746597925822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2560,0.024151466290156045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3072,0.026374399662017822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,4096,0.030846933523813885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,3584,0.02889813383420308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,2048,0.022691200176874794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1536,0.020830933252970377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,1024,0.018981333573659262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,768,0.016459733247756958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,512,0.01548373301823934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,256,0.015200000007947287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,128,0.01518186628818512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,64,0.01520639955997467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,32,0.015392000476519266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,10240,65536,0.26916160583496096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,12288,0.051829334100087485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,10240,0.04589440027872722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,16384,0.0634879986445109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,8192,0.043287467956542966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,7168,0.036541867256164554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,5120,0.03075946569442749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,6144,0.03393919865290324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3072,0.02391466697057088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,3584,0.02529919942220052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2560,0.022357332706451415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,4096,0.027128533522288008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,2048,0.02085226575533549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1024,0.01783039967219035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,1536,0.01957119901974996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,768,0.01597653329372406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,512,0.015994667013486227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,256,0.015476266543070475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,128,0.01527679959932963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,64,0.015428266922632852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,32,0.015485866864522298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,8192,65536,0.2298250675201416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,12288,0.04820053180058797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,10240,0.042439464728037515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,16384,0.05857173204421997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,8192,0.040642134348551434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,4096,0.02579840024312337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,7168,0.03446400165557861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,6144,0.0316106657187144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,5120,0.028940800825754804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3584,0.02437973419825236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,3072,0.023013333479563393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2560,0.021606399615605672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,2048,0.020566399892171225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1536,0.019351466496785482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,768,0.01608746647834778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,1024,0.016426666577657064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,512,0.015871999661127727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,256,0.015499732891718545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,128,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,64,0.01525973379611969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,32,0.015340800086657206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,7168,65536,0.20068480173746744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,8192,0.03552533388137817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,12288,0.04372053146362305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,10240,0.039331201712290445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,16384,0.052515200773874914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,4096,0.024476800362269083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,5120,0.026708267132441205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,7168,0.03189226587613424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,6144,0.029028266668319702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3584,0.02336853345235189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,3072,0.022100265820821127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2560,0.02090453306833903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,2048,0.019845332702000937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1536,0.018806399901707967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,768,0.015991466244061787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,512,0.01569066643714905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,1024,0.01581760048866272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,256,0.015270400047302245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,65536,0.1784832000732422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,128,0.01502826710542043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,64,0.01511573294798533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,6144,32,0.015011200308799743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,8192,0.031221334139506025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,10240,0.035334400335947674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,12288,0.039851733048756915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,16384,0.04714560111363729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,6144,0.02658240000406901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,5120,0.024861866235733034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,4096,0.022703999280929567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,7168,0.029269333680470782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3072,0.021134932835896812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,3584,0.021915733814239502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2560,0.02001919945081075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,2048,0.01908373236656189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1536,0.017672532796859743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,768,0.015795200069745382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,1024,0.015862400333086647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,65536,0.14778560002644855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,512,0.015532799561818442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,256,0.015412267049153647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,128,0.01511146624883016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,64,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,5120,32,0.015203199783960977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,10240,0.03204266627629598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,8192,0.02858453392982483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,12288,0.035300266742706296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,16384,0.04523200194040934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,7168,0.026308266321818034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,4096,0.022507733106613158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,6144,0.024985599517822265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,5120,0.02305813431739807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3072,0.019862399498621622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,3584,0.020730666319529214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2560,0.01924906571706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,2048,0.018126932779947917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1536,0.016060800353686015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,1024,0.01588159998257955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,65536,0.12478400071461995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,768,0.015715199708938598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,512,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,256,0.015051733454068503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,128,0.014889599879582724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,64,0.014904533823331198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,4096,32,0.014829867084821067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,8192,0.02644053300221761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,16384,0.04328960180282593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,10240,0.029780266682306926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,12288,0.03689066569010417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,7168,0.02487466732660929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,6144,0.023414399226506552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,5120,0.02212160031000773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,4096,0.020936532815297445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3584,0.020148267348607383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,3072,0.019522132476170857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,65536,0.11252693335215251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2560,0.018994132677714028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,2048,0.01707093318303426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1536,0.015987199544906617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,1024,0.015953066945075988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,768,0.01570453345775604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,512,0.01546239952246348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,256,0.014998400211334228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,64,0.01495573321978251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,128,0.014929067095120749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3584,32,0.015050666530927024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,10240,0.02760319908459981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,8192,0.024796799818674723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,16384,0.03884266614913941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,12288,0.03109440008799235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,6144,0.022545067469278972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,7168,0.023948800563812257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,4096,0.02044373353322347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,5120,0.021309866507848104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3584,0.019610667228698732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2560,0.018334933121999106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,3072,0.01900586684544881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,2048,0.016770132382710776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,65536,0.1033141295115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1536,0.01580693324406942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,1024,0.01591253379980723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,768,0.01568000018596649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,512,0.01536853313446045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,256,0.015210666259129844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,128,0.014916266997655234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,64,0.015032533804575601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,3072,32,0.014909866452217101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,12288,0.02871999939282735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,10240,0.0262880007425944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,8192,0.023766400416692098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,16384,0.035964798927307126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,7168,0.023063466946283976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,6144,0.02186346650123596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,4096,0.019700266917546592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,5120,0.020937599738438926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3584,0.019032533963521323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,65536,0.08957546552022298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,3072,0.018552533785502114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2560,0.01766080061594645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,2048,0.01680213411649068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1536,0.016076800227165223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,1024,0.01572480003039042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,768,0.015588266650835672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,128,0.014909866452217101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,512,0.015556266903877259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,256,0.015043200055758158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,32,0.014859732985496522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2560,64,0.014947199821472168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,12288,0.027988266944885255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,16384,0.03256426652272542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,10240,0.026263467470804852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,8192,0.0243776003519694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,7168,0.02369813323020935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,5120,0.02030186653137207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,6144,0.021229867140452066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,4096,0.01890666683514913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,65536,0.07830719947814942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3584,0.018644267320632936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,3072,0.017517866690953572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2560,0.016609066724777223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1536,0.01632426679134369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,2048,0.016269866625467935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,1024,0.015615999698638916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,768,0.015473066767056783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,256,0.015095466375350952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,512,0.015296000242233276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,128,0.014829867084821067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,64,0.01477013329664866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,2048,32,0.014816000064214071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,16384,0.0291103998819987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,12288,0.025613866249720257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,8192,0.02324159940083822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,10240,0.024320000410079957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,7168,0.02149333357810974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,6144,0.02057386636734009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,5120,0.018947199980417887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,65536,0.06837546825408936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,4096,0.017926400899887084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3584,0.01723840037981669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,3072,0.016701867183049522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2560,0.01689280072848002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1536,0.016125866770744325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,2048,0.016545066237449647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,1024,0.015846400459607442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,768,0.015442132949829102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,512,0.015343999862670899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,256,0.01492800017197927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,128,0.014798933267593383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,64,0.014898133277893067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1536,32,0.014758400122324624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,16384,0.02584106723467509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,12288,0.02424959937731425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,10240,0.023386667172114052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,8192,0.022179200251897176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,6144,0.02084160049756368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,7168,0.021594667434692384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,65536,0.05655680100123087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,5120,0.018709333737691243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,4096,0.01807039976119995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3584,0.017356799046198527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2560,0.01687999963760376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,3072,0.017129600048065186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,2048,0.016377600034077962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1536,0.01609599987665812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,1024,0.015702399611473083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,768,0.015558399756749473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,512,0.015179733435312906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,256,0.014856533209482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,128,0.015027200182278952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,64,0.015107199549674988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,1024,32,0.014859732985496522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,16384,0.024995199839274087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,12288,0.023057067394256593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,10240,0.021894399325052896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,8192,0.020219733317693077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,65536,0.05084586540857951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,6144,0.019731199741363524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,7168,0.020334933201471964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,5120,0.01876373291015625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,4096,0.01793173352877299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3584,0.017553067207336424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,3072,0.017084799210230508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1536,0.01606826682885488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2048,0.016394666830698647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,2560,0.016710400581359863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,1024,0.015522133310635886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,768,0.01530026694138845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,512,0.015190399686495461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,256,0.01492693324883779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,128,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,32,0.014693333705266317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,768,64,0.014658133188883463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,16384,0.022964266935984294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,12288,0.02023893396059672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,65536,0.044232531388600664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,10240,0.020275199413299562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,8192,0.019627734025319418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,7168,0.020615466435750327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,6144,0.019592533508936562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,4096,0.017883733908335368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,5120,0.01882560054461161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3584,0.01732586622238159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,3072,0.0170741339524587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2560,0.016598400473594666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,2048,0.01621333360671997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1536,0.016029866536458333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,768,0.015501866738001505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,1024,0.015633066495259605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,256,0.014797866344451904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,128,0.014788267016410828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,512,0.015156267086664834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,64,0.014802133043607077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,512,32,0.014678399761517844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,16384,0.020966400702794395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,12288,0.020411733786265054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,10240,0.020427733659744263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,65536,0.0355679988861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,8192,0.01997013290723165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,7168,0.02021333376566569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,5120,0.01869866649309794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,4096,0.017935999234517417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,6144,0.01994880040486654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3584,0.017427200078964235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,3072,0.017203199863433837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2560,0.01672533353169759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,2048,0.016339199741681416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1536,0.016082132856051125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,768,0.015544533729553223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,1024,0.01554026703039805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,512,0.015103999773661295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,256,0.014801067113876343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,128,0.014822399616241455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,64,0.014717866977055868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,256,32,0.01467519998550415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,65536,0.03214933276176453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,16384,0.021013333400090536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,10240,0.02031893332799276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,12288,0.020376533269882202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,8192,0.019843200842539467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,7168,0.02027413249015808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,5120,0.018807466824849448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,6144,0.01949653426806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,4096,0.017966934045155845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3584,0.017552000284194947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,3072,0.017167999347050985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2560,0.01664959987004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,2048,0.016422399878501893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1536,0.01591146687666575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,1024,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,768,0.015432533621788026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,512,0.015200000007947287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,256,0.014724266529083253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,128,0.01476479967435201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,64,0.01469546655813853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,128,32,0.014668800433476768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,16384,0.020691200097401937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,12288,0.019973333676656088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,65536,0.029742934306462604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,8192,0.019350399573644005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,10240,0.020182400941848755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,6144,0.01949653426806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,7168,0.019988266626993816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,5120,0.018786134322484334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,4096,0.01797653237978617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3072,0.016973867019017538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,3584,0.01728640000025431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2560,0.016618667046229045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,2048,0.01627306640148163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1536,0.015818666418393454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,1024,0.015590399503707886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,768,0.015126400192578635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,256,0.014823466539382935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,512,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,128,0.014643200238545737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,64,0.014640000462532044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,64,32,0.014465066790580749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,16384,0.020584533611933388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,65536,0.02980159918467204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,12288,0.019947733481725058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,10240,0.02018666664759318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,8192,0.01934400002161662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,7168,0.01985599994659424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,5120,0.018552533785502114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,6144,0.01928960084915161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,4096,0.017773866653442383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3584,0.017313067118326822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2560,0.01656000018119812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,3072,0.016796799500783284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,2048,0.01644053359826406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1536,0.015913599729537965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,768,0.015520000457763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,256,0.014849066734313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,1024,0.015416533748308817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,512,0.015065600474675497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,128,0.014755200346310934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,64,0.014645333091417948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,64,32,32,0.014737066626548768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,7168,0.17698346773783366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,8192,0.2001866658528646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,10240,0.24349973996480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,12288,0.2918773333231608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,5120,0.13192213376363118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,6144,0.15472426414489746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,4096,0.10851519902547199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,16384,0.38031253814697263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3584,0.09843839804331461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2560,0.0752021312713623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1536,0.05890239874521891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,1024,0.041732267538706465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,3072,0.08660053412119548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,2048,0.06356159845987955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,128,0.019658666849136353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,256,0.022137600183486938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,512,0.028780800104141236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,768,0.03592853148778279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,64,0.017842133839925133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,65536,32,0.01779200037320455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,8192,0.061476266384124754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,10240,0.07228906949361166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,12288,0.08414293130238851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,16384,0.10550399621327718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,5120,0.04456213315327962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,7168,0.055837865670522055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,6144,0.050222933292388916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,4096,0.03891093333562215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3584,0.03588266770044963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,3072,0.03320853312810262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2560,0.030348799626032513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,2048,0.027178666989008587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1536,0.02379946708679199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,1024,0.020901334285736085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,768,0.019671465953191122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,512,0.018180267016092936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,128,0.01569706698258718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,256,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,64,0.015613866845766702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,32,0.015774933497111003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,10240,0.059639465808868405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,12288,0.07072640260060628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,16384,0.0887061357498169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,8192,0.051218132177988686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,6144,0.04341439803441365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,7168,0.04820160071055095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,16384,65536,0.37203734715779624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,4096,0.03431040048599243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3584,0.03177173336346944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,5120,0.03792746861775716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,3072,0.03146880070368449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2560,0.026871466636657716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1536,0.021955200036366782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,2048,0.024285866816838583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,768,0.018796799580256145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,1024,0.019640533129374187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,512,0.016310399770736693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,256,0.01578986644744873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,128,0.015528532862663268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,64,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,32,0.015294933319091797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,12288,65536,0.2964586575826009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,10240,0.05256426731745402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,12288,0.05927040179570516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,16384,0.07370879650115966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,8192,0.04936213493347168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,7168,0.04169066747029622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,5120,0.034373335043589276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,6144,0.03770560026168823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,4096,0.030261333783467608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3072,0.02624959945678711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2560,0.024229333798090617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,3584,0.028538666168848675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,2048,0.022539732853571574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1536,0.02070186734199524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,1024,0.018935465812683107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,512,0.016403200229008992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,768,0.017406932512919106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,256,0.015579733252525329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,128,0.015242666999499003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,64,0.0153546671072642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,32,0.015284267067909241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,10240,65536,0.2593279997507731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,12288,0.05076160033543905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,10240,0.0452949325243632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,16384,0.062116265296936035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,8192,0.04277226527531942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,6144,0.03332800070444743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,7168,0.03612693150838216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,5120,0.030105600754419964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,4096,0.027129600445429485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3072,0.023702400922775268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2560,0.02230613430341085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,3584,0.025498666365941364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,2048,0.02140053311983744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1536,0.01967786749204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,1024,0.01772159934043884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,768,0.016501333316167197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,256,0.015481600165367126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,512,0.015925332903862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,128,0.015357866883277893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,64,0.015252266327540079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,32,0.015379200379053751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,8192,65536,0.20658346811930337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,8192,0.03935360113779704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,12288,0.047304534912109376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,16384,0.05723626613616943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,10240,0.042078932126363114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,4096,0.0255840003490448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,5120,0.028674133618672687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,7168,0.03404693206151326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,6144,0.03171520034472148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3584,0.024333866437276204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,3072,0.02318293253580729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2560,0.02180160085360209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,2048,0.020245333512624107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1536,0.01907093326250712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,1024,0.017038933436075845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,768,0.015672533710797628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,512,0.015396266182263692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,256,0.015278933445612588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,128,0.015094400445620219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,64,0.015102932850519816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,32,0.0150218665599823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,7168,65536,0.20090773900349937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,12288,0.044809599717458085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,8192,0.03612266778945923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,16384,0.053430398305257164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,10240,0.03880106608072917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,5120,0.027379200855890913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,4096,0.024526933828989662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,7168,0.03272426724433899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,6144,0.02996266682942708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3584,0.0234826664129893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2560,0.02122559944788615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,3072,0.02208426594734192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,2048,0.019789866606394448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1536,0.01876586675643921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,1024,0.016140799721082053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,512,0.015610667069753012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,768,0.015709867080052696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,65536,0.16602986653645832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,128,0.015210666259129844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,64,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,256,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,6144,32,0.015014400084813436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,8192,0.031038933992385866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,12288,0.03893653154373169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,10240,0.034669868151346844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,16384,0.04632426500320434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,6144,0.026636799176534016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,5120,0.024476800362269083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,7168,0.028943999608357744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,4096,0.022813866535822548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3584,0.021874133745829263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2048,0.019139200448989868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,2560,0.02004693349202474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,3072,0.020670932531356812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1536,0.017655466000239053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,1024,0.01602240006128947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,512,0.01548693378766378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,768,0.015585066874821982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,65536,0.14192214012145996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,256,0.015247999628384908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,128,0.015090133746465048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,64,0.015046399831771851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,5120,32,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,10240,0.03147520025571187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,12288,0.03463253180185954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,8192,0.027447466055552167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,16384,0.043738667170206705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,6144,0.02435520092646281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,7168,0.02608213424682617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,4096,0.021451733509699502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,5120,0.022657066583633423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3584,0.020594133933385213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2560,0.019220266739527384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,3072,0.019797333081563315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,2048,0.018062933286031087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1536,0.015783466895421348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,65536,0.11909759839375814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,1024,0.015682133038838704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,768,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,512,0.015240533153216043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,256,0.0150026669104894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,128,0.014873600006103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,64,0.01493333379427592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,4096,32,0.015090133746465048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,8192,0.02593280076980591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,12288,0.032605866591135665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,16384,0.04102826515833537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,10240,0.02937920093536377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,7168,0.024596265951792397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,6144,0.02350613276163737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,5120,0.022233599424362184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,4096,0.02055466572443644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3584,0.020107734203338622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,3072,0.01938026746114095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2560,0.018873600165049235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,65536,0.1065621296564738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,2048,0.01686826745669047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1536,0.01637226641178131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,1024,0.016011733810106912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,768,0.015687466661135355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,512,0.015227733055750528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,256,0.014949333667755128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,128,0.014855466286341348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,64,0.014894933501879374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3584,32,0.014990933736165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,10240,0.03041599988937378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,8192,0.02820906639099121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,12288,0.03335573275883992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,16384,0.039250131448109946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,6144,0.022637865940729775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,7168,0.02364586591720581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,5120,0.021272534132003786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,4096,0.020331732432047524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3584,0.019646932681401573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,3072,0.018894932667414346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,65536,0.09795200030008952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2560,0.017757866779963175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,2048,0.01646293302377065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1536,0.01618559956550598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,1024,0.01596799989541372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,512,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,768,0.015638400117556253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,256,0.014964266618092855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,128,0.014847999811172486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,64,0.014849066734313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,3072,32,0.01487573285897573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,12288,0.030087467034657794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,16384,0.03497813145319621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,10240,0.02804906765619914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,8192,0.025563732782999678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,7168,0.024805333216985068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,6144,0.021708800395329794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,5120,0.020615466435750327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,4096,0.019665066401163736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3584,0.01922666629155477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,65536,0.0877791961034139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2560,0.017941333850224814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,3072,0.01788160006205241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,2048,0.016472533345222473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1536,0.016167466839154564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,768,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,1024,0.015713066856066386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,512,0.015457066893577575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,256,0.014968533317248026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,128,0.014873600006103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,64,0.014775466918945313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2560,32,0.014807466665903726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,12288,0.02757973273595174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,10240,0.026039467255274458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,16384,0.03127786715825399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,8192,0.02461120088895162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,7168,0.023627734184265135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,6144,0.021167999505996703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,5120,0.020113066832224528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,4096,0.019208532571792603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3584,0.017857066790262856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,65536,0.07422826290130616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,3072,0.01699733336766561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2560,0.016620799899101257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1536,0.016335999965667723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,2048,0.016505600015322367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,768,0.015452800194422403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,1024,0.015795200069745382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,512,0.015212800105412802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,256,0.015015467007954916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,64,0.014793599645296732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,128,0.014738133549690247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,2048,32,0.01472106675306956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,16384,0.027903999884923297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,12288,0.025226666529973345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,10240,0.023965867360432942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,8192,0.02195840080579122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,7168,0.02118826707204183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,6144,0.020454400777816774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,5120,0.01924053430557251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,4096,0.0181877334912618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,65536,0.06365546782811483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3072,0.017061332861582436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,3584,0.01757546663284302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2560,0.016595199704170227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,2048,0.01643946667512258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1536,0.01590933303038279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,768,0.015422933300336204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,1024,0.015784533818562825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,512,0.015175466736157736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,256,0.014849066734313965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,64,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,128,0.014813866217931113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1536,32,0.014803199966748556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,12288,0.0231221338113149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,10240,0.022844799359639487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,8192,0.021926399072011313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,16384,0.025117866198221844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,65536,0.05220160086949667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,7168,0.021242666244506835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,5120,0.01877440015474955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,4096,0.01800959904988607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,6144,0.019992534319559732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3584,0.017643733819325765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,3072,0.016951467593510947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2560,0.016884267330169678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,2048,0.01642346680164337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1536,0.016038399934768677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,1024,0.01577279965082804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,512,0.015202132860819497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,768,0.01548799971739451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,256,0.014912000298500061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,128,0.014823466539382935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,64,0.014716800053914389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,1024,32,0.014756266276041666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,16384,0.02410773237546285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,12288,0.023513599236806234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,8192,0.021398399273554483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,10240,0.022398932774861654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,65536,0.04666453202565511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,7168,0.02015786568323771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,6144,0.019766400257746376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,5120,0.018411733706792197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3584,0.017523199319839478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,4096,0.017767467101415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2048,0.016335999965667723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,3072,0.017105066776275636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1536,0.01618133286635081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,2560,0.01681173245112101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,1024,0.015602133671442666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,768,0.015284267067909241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,512,0.015224533279736838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,256,0.014961066842079162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,128,0.014748799800872802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,64,0.014708266655604044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,768,32,0.014685866236686707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,16384,0.02332373261451721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,12288,0.020594133933385213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,10240,0.020472532510757445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,65536,0.04001813332239787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,8192,0.01984106699625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,7168,0.020004266500473024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,6144,0.019399466117223103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,4096,0.01771093408266703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,5120,0.018484266599019368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3584,0.01745706597963969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,3072,0.016935465733210246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2560,0.01662506659825643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,2048,0.016402133305867515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1536,0.01583146651585897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,1024,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,768,0.015333333611488342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,512,0.01511679987112681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,256,0.014845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,64,0.014669866363207499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,128,0.014570666352907815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,512,32,0.014693333705266317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,16384,0.020246400435765585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,65536,0.03364053169886271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,12288,0.01988906661669413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,10240,0.01999893387158712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,8192,0.019526400168736777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,7168,0.019990400473276774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,5120,0.018547199169794717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,4096,0.017537067333857216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,6144,0.019627734025319418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3584,0.01731946667035421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,3072,0.0169322669506073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2560,0.016808533668518068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,2048,0.016210132837295534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1536,0.01581760048866272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,1024,0.015566933155059814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,768,0.015287466843922935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,256,0.01476800044377645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,512,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,128,0.014688000082969666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,64,0.014647466937700906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,256,32,0.014562132954597472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,16384,0.020499199628829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,65536,0.029297065734863282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,12288,0.0198527991771698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,10240,0.020108799139658608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,8192,0.01973759929339091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,7168,0.02022613286972046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,5120,0.018525866667429607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,4096,0.017638399203618368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,6144,0.019475199778874717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3584,0.01733013391494751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,3072,0.01683626572291056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2560,0.016708266735076905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,2048,0.016302933295567833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1536,0.01583573321501414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,1024,0.01574720044930776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,512,0.015105066696802774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,768,0.015266133348147073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,256,0.014840533336003622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,128,0.014617600043614707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,64,0.014698666334152222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,128,32,0.014633599917093912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,16384,0.020577067136764528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,65536,0.026852265993754072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,12288,0.019824000199635823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,8192,0.01926079988479614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,10240,0.020166399081548055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,7168,0.01994453271230062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,6144,0.019433599710464478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,5120,0.018458666404088338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,4096,0.017875200510025023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3584,0.01731520096460978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,3072,0.016945066054662068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2560,0.01653439998626709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,2048,0.01620266636212667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1536,0.01581333378950755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,768,0.01544426679611206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,1024,0.015451733271280924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,512,0.015030399958292643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,256,0.0147189329067866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,64,0.014595199624697366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,128,0.014723199605941772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,64,32,0.014579199751218162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,65536,0.02649066646893819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,16384,0.020486400524775187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,12288,0.01990399956703186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,10240,0.019748266537984213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,8192,0.019425066312154134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,7168,0.019723733266194664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,5120,0.018447999159495035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,6144,0.019232000907262167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,4096,0.01753386656443278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3584,0.017230933904647826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,3072,0.016898133357365928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2560,0.016812799374262492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,2048,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1536,0.0157258669535319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,1024,0.015526400009791056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,768,0.015251200397809347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,512,0.01507306694984436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,256,0.014656000336011252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,128,0.014605866869290671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,64,0.014537599682807923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,48,32,32,0.014643200238545737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,7168,0.1750208059946696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,8192,0.19737706184387208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,10240,0.2439626693725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,12288,0.28790079752604164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,6144,0.15287680625915528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,5120,0.12974826494852704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,4096,0.10764373143513997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,16384,0.3827925364176432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3584,0.09636800289154053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1536,0.057835733890533446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2560,0.07415573596954346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,3072,0.08537493546803793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,1024,0.04025599956512451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,2048,0.06345706780751546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,128,0.01936639944712321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,256,0.02150613268216451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,64,0.0173962672551473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,512,0.028061866760253906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,768,0.03476053476333618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,65536,32,0.017590399583180746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,8192,0.06070506572723389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,10240,0.07165546417236328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,12288,0.08325973351796469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,16384,0.10546027024586994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,6144,0.049600001176198324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,5120,0.04421866734822591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,7168,0.0549888014793396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,4096,0.038318932056427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2560,0.030034132798512775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3072,0.03302826682726542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,3584,0.035847465197245285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,2048,0.026811732848485308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1024,0.020849066972732543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,768,0.01949333349863688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,1536,0.02362346649169922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,512,0.017820799350738527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,256,0.01557866632938385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,128,0.015448533495267234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,64,0.015383467078208923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,32,0.015413332978884378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,10240,0.06054933468500773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,12288,0.06803092956542969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,16384,0.08896533648173013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,8192,0.051863467693328856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,7168,0.046199464797973634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,6144,0.042984533309936526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,16384,65536,0.3648789405822754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,4096,0.03307733337084452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3584,0.0313813328742981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,5120,0.038114134470621744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,3072,0.030766934156417847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2560,0.02600319981575012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,2048,0.0240447998046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1536,0.021597866217295328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,1024,0.019399466117223103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,768,0.0182751993338267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,512,0.015702399611473083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,256,0.015425067146619162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,128,0.015282133221626281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,64,0.01516480048497518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,32,0.015290666619936624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,12288,65536,0.28945706685384115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,10240,0.0521727999051412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,12288,0.05962560176849365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,16384,0.07321706612904867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,8192,0.048622934023539226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,5120,0.03379520177841187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,6144,0.037291733423868816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,7168,0.041060264905293783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3072,0.026055467128753663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2560,0.024165334304173787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,3584,0.027934932708740236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,4096,0.02986666758855184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,2048,0.022296533981959025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1536,0.02074880003929138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,1024,0.018888533115386963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,768,0.017525333166122436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,512,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,256,0.015367466211318969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,128,0.015042133132616677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,64,0.015252266327540079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,32,0.0151146670182546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,10240,65536,0.25196906725565593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,10240,0.044707198937733963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,12288,0.050271999835968015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,16384,0.06111893256505331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,8192,0.04225279887517293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,7168,0.03556480010350545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,5120,0.029743999242782593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,6144,0.03261546691258748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,4096,0.026693334182103474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3584,0.02510400017102559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,3072,0.023596799373626708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2560,0.02211946646372477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1536,0.01952426632245382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,2048,0.020683733622233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,1024,0.01780479947725932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,768,0.01576533317565918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,512,0.015698132912317912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,256,0.015380266308784484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,128,0.015093333522478738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,64,0.015058133006095886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,32,0.01520746648311615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,8192,65536,0.20162240664164224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,8192,0.03893440167109172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,12288,0.04660586516062419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,10240,0.04129600127538045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,16384,0.05594559907913208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,4096,0.02529279987017314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,6144,0.0310805340607961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,7168,0.03354453245798747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,5120,0.028036266565322876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3584,0.024195200204849242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2560,0.021709867318471274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,3072,0.022988800207773843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,2048,0.02016213337580363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1536,0.019076265891393027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,1024,0.01693120002746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,768,0.015662933389345803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,512,0.015331199765205384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,256,0.015198933084805808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,128,0.015020799636840821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,64,0.015034666657447815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,32,0.014865066607793173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,7168,65536,0.18996159235636395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,8192,0.03567146857579549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,12288,0.04351679881413777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,16384,0.05295360088348389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,10240,0.0383135994275411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,5120,0.02709653377532959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,4096,0.02434239983558655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,7168,0.03201280037562053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,6144,0.02954453428586324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3584,0.02358400026957194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,3072,0.02206933299700419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2560,0.020963199933369956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,2048,0.0196234663327535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1536,0.018402133385340372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,1024,0.016646400094032288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,768,0.015518933534622192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,512,0.015362133582433065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,256,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,65536,0.1589194615681966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,128,0.014919466773668923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,64,0.014995200435320535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,6144,32,0.014920533696810404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,12288,0.038177065054575604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,10240,0.03412053187688192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,16384,0.04496106704076131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,8192,0.03036053379376729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,6144,0.026184533039728803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,5120,0.024371200799942018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,7168,0.02871359984079997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,4096,0.022434133291244506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3584,0.02146986722946167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2560,0.020052266120910645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,3072,0.020845866203308104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,2048,0.01890986760457357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1024,0.01548479994138082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,1536,0.017297067244847617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,512,0.015270400047302245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,768,0.015495466192563376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,65536,0.13763200441996257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,256,0.015089066823323569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,64,0.014827733238538107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,128,0.014908799529075622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,5120,32,0.014934399724006652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,8192,0.02720746596654256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,12288,0.034414935111999514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,10240,0.030730666716893514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,16384,0.04260480006535848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,7168,0.02545493245124817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,6144,0.024293333292007446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,5120,0.022827732563018798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,4096,0.021074134111404418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3584,0.020496000846227012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,3072,0.01990293264389038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2560,0.018969599405924478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,2048,0.01786880095799764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,65536,0.11420053641001385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1536,0.016125866770744325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,1024,0.015753600001335143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,768,0.015545599659283958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,512,0.015069866180419922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,256,0.015032533804575601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,128,0.014882133404413859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,64,0.015000533064206442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,4096,32,0.01513706644376119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,12288,0.03192960023880005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,16384,0.03955519994099935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,10240,0.0316106657187144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,8192,0.025895466407140095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,7168,0.02447893420855204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,6144,0.022983467578887938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,5120,0.021720532576243082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,4096,0.020802134275436403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3584,0.020057600736618043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,3072,0.019516799847284952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2560,0.018573866287867228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,2048,0.017854932943979898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,65536,0.10223253568013509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1536,0.01615466674168905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,1024,0.015727999806404113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,512,0.01516266663869222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,768,0.01544533371925354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,256,0.014980266491572062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,128,0.014885333180427552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,64,0.014938666423161825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3584,32,0.014713600277900696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,10240,0.029864533742268877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,12288,0.03242666721343994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,16384,0.03746453523635864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,8192,0.027672533194224042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,7168,0.02341866691907247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,6144,0.022107734282811483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,5120,0.021040000518163047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,4096,0.01979413429896037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3584,0.019388800859451293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,3072,0.018956800301869713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2560,0.017333332697550455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,65536,0.09370880126953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,2048,0.016307199994723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1536,0.016105600198109946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,768,0.015490133563677469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,1024,0.015672533710797628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,512,0.01515733301639557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,256,0.014885333180427552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,128,0.014814933141072592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,64,0.014814933141072592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,3072,32,0.014816000064214071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,12288,0.029531733194986982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,16384,0.0336575984954834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,10240,0.027180800835291546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,8192,0.025382399559020996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,7168,0.024284799893697105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,6144,0.02338026762008667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,4096,0.019299199183781944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,5120,0.020413867632548013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,65536,0.08126826286315918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3584,0.018986666202545167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2560,0.016410666704177856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,2048,0.016402133305867515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,3072,0.01789120038350423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1024,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,1536,0.01603626708189646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,768,0.01534293293952942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,512,0.015172266960144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,256,0.01495253344376882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,128,0.01480959951877594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,64,0.014747732877731323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2560,32,0.01477120021979014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,12288,0.026985599597295122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,10240,0.02534186641375224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,16384,0.03073173364003499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,8192,0.023907200495402018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,7168,0.023246934016545616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,5120,0.019966934124628702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,6144,0.02230186661084493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,4096,0.018755199511845906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3584,0.017413334051767985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,65536,0.06922880013783773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,3072,0.0168287992477417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2560,0.01670080025990804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,2048,0.016314666469891867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1536,0.016033066312472026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,1024,0.015493333339691162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,768,0.015479466319084168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,512,0.015265066425005594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,256,0.014968533317248026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,128,0.014905599753061929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,64,0.014597333470980325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,2048,32,0.014629333217938741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,12288,0.025014400482177734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,16384,0.027031467358271284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,10240,0.023577600717544556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,8192,0.022454400857289634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,7168,0.022081067164738975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,6144,0.02140053311983744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,5120,0.01917546590169271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,65536,0.05942399899164835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3584,0.017084799210230508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,4096,0.017486933867136636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,3072,0.016758400201797485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2560,0.016664533813794454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,2048,0.01649173299471537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1536,0.016099199652671814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,1024,0.015568000078201295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,768,0.015361066659291586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,512,0.015179733435312906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,128,0.014708266655604044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,256,0.014840533336003622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,64,0.014689067006111145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1536,32,0.01474666694800059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,16384,0.02452053427696228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,12288,0.022443733612696328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,8192,0.021241599321365358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,10240,0.02198293407758077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,65536,0.04800320068995158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,7168,0.02062506675720215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,6144,0.01956160068511963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,5120,0.017973333597183228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,4096,0.017478400468826295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3584,0.017422932386398315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,3072,0.016910932461420693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2560,0.016590933005015053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,2048,0.016338133811950685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1536,0.015783466895421348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,1024,0.01546346644560496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,768,0.015369600057601929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,512,0.01502293348312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,256,0.014847999811172486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,128,0.01474346617857615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,64,0.014703999956448874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,1024,32,0.014669866363207499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,16384,0.0230240007241567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,12288,0.02137813369433085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,10240,0.021256534258524577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,8192,0.019189333915710448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,65536,0.04240213235219319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,6144,0.019511467218399046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,7168,0.019810134172439577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,4096,0.017565866311391197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,5120,0.018183465798695883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3584,0.01728960076967875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,3072,0.016986666123072307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2560,0.016530133287111917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,2048,0.016361600160598753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1536,0.01590933303038279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,1024,0.015465600291887918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,768,0.015347199638684592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,512,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,256,0.015030399958292643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,128,0.014683733383814493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,32,0.014636799693107605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,768,64,0.014573867122332254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,16384,0.021983999013900756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,65536,0.03574719826380412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,10240,0.020177066326141357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,12288,0.020719999074935914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,8192,0.0192138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,7168,0.019638399283091225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,6144,0.01941546599070231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,5120,0.0181386669476827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,4096,0.01738133430480957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3072,0.016985599199930826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,3584,0.017450666427612303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2560,0.01664426624774933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,2048,0.016199466586112977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1536,0.01581439971923828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,1024,0.015500799814860026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,768,0.015381333231925965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,256,0.014841600259145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,512,0.015127467115720114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,64,0.014677332838376364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,128,0.014600533246994018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,512,32,0.014729600151379904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,16384,0.02021119991938273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,65536,0.029028266668319702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,12288,0.0196234663327535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,10240,0.019831466674804687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,8192,0.01944640080134074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,7168,0.01986560026804606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,5120,0.01827413241068522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,4096,0.017513600985209148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,6144,0.01922666629155477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3584,0.017153066396713258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2560,0.016706132888793947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,3072,0.017043199141820273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,2048,0.016426666577657064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1536,0.01575573285420736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,1024,0.015557333827018738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,768,0.015330132842063905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,512,0.015063466628392539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,256,0.014774399995803832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,128,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,32,0.014658133188883463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,256,64,0.014646400014559427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,65536,0.026396799087524413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,16384,0.019990400473276774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,12288,0.019307732582092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,10240,0.01973653237024943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,8192,0.019143466154734293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,7168,0.01956160068511963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,6144,0.0191594660282135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,5120,0.018217599391937254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3584,0.017478400468826295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,4096,0.01772159934043884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,3072,0.016988799969355265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2048,0.01611306667327881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,2560,0.016661333044370015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1536,0.015745066603024802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,1024,0.01550933321317037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,768,0.01525759994983673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,128,0.014693333705266317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,512,0.015007999539375306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,256,0.014818132917086283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,64,0.014728533228238425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,128,32,0.014632532993952433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,10240,0.019720532496770225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,16384,0.019802665710449217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,65536,0.02488213380177816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,12288,0.019355734189351402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,8192,0.019108267625172932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,7168,0.019537067413330077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,5120,0.01811520059903463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,6144,0.019107200702031455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,4096,0.017526400089263917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3584,0.01723946730295817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,3072,0.016809600591659545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2560,0.01641493340333303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,2048,0.016184533635775246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1536,0.015742933750152587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,1024,0.015396266182263692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,768,0.015223466356595359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,512,0.01504853367805481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,256,0.014805333813031516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,64,0.014607999722162882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,128,0.014516266187032065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,64,32,0.014564266800880432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,12288,0.019432532787322997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,16384,0.019828265905380248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,65536,0.025169066588083905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,10240,0.019451733430226645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,8192,0.019035732746124266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,7168,0.019476266702016194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,6144,0.0189301331837972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,5120,0.018284799655278523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3584,0.01717546582221985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,4096,0.017552000284194947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,3072,0.01681173245112101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2560,0.016428800423940022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,2048,0.016264533003171287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1536,0.015836800138155617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,1024,0.015268266201019287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,768,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,512,0.0149536003669103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,128,0.014637866616249084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,256,0.014851199587186179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,64,0.01458453337351481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,32,32,32,0.01461120049158732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,7168,0.17308905919392903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,8192,0.19739519755045573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,10240,0.239080540339152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,12288,0.28544534047444664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,5120,0.1282687981923421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,6144,0.15116267204284667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,4096,0.10538346767425537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,16384,0.38037331899007165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3584,0.09527466297149659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2560,0.07284159660339355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,3072,0.08403626283009848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,2048,0.061001598834991455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1024,0.039128533999125165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,1536,0.056576001644134524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,512,0.027370667457580565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,256,0.021163733800252278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,768,0.033479468027750654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,128,0.018556799491246542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,64,0.0165994664033254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,65536,32,0.01689493258794149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,8192,0.05949653387069702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,10240,0.070470396677653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,12288,0.08178026676177978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,16384,0.10280319849650066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,5120,0.0437450647354126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,6144,0.04897600015004476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,7168,0.054041600227355956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,4096,0.03775360186894734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2560,0.029597866535186767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3584,0.03524693250656128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,3072,0.03226666649182637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,2048,0.026232532660166424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1536,0.023475199937820435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,1024,0.020601600408554077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,768,0.01930239995320638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,512,0.017547732591629027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,256,0.015526400009791056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,128,0.015090133746465048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,64,0.015337600310643514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,32,0.015230933825174967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,10240,0.05788266658782959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,12288,0.06618239879608154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,16384,0.08315093517303467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,8192,0.049396268526713055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,7168,0.044880000750223796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,6144,0.04139093160629272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,16384,65536,0.35838292439778646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,4096,0.0325055996576945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,5120,0.036713600158691406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3584,0.030326400200525922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,3072,0.02999253273010254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2560,0.02549546758333842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1536,0.021590399742126464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,2048,0.023536000649134317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,1024,0.019357866048812865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,768,0.018244266510009766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,512,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,256,0.015433599551518759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,128,0.014988799889882406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,64,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,32,0.0151936004559199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,12288,65536,0.2834954579671224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,10240,0.0508565346399943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,12288,0.057150932153066006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,16384,0.07151573499043783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,8192,0.04749973217646281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,7168,0.040223999818166094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,6144,0.03647679885228475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,5120,0.033138134082158405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3072,0.025510400533676147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,4096,0.02935360074043274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,3584,0.027587199211120607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2560,0.02368639906247457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,2048,0.02190613349278768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1536,0.020357332626978555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,1024,0.01862506667772929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,768,0.01691733400026957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,512,0.015413332978884378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,256,0.015056000153223673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,128,0.015019733707110086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,64,0.015046399831771851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,32,0.014987732966740927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,10240,65536,0.23762772878011068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,12288,0.049388798077901204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,10240,0.043321601549784344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,16384,0.05969280004501343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,8192,0.041220267613728837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,5120,0.02956906755765279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,7168,0.035138134161631265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,6144,0.03236906727155049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,4096,0.026315732796986895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2560,0.02215893268585205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3072,0.023520000775655112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,3584,0.024915200471878052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,2048,0.02058666745821635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1024,0.01760106682777405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,768,0.015762133399645488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,1536,0.019382399320602418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,512,0.01572480003039042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,128,0.014932266871134438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,256,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,64,0.014975999792416891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,32,0.015118933717409768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,8192,65536,0.19400854110717775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,10240,0.04023573398590088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,12288,0.0454090674718221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,8192,0.038466131687164305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,16384,0.05459200143814087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,4096,0.025065600872039795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,7168,0.033062400420506795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,6144,0.030822400252024335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,5120,0.028017065922419232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3584,0.02390506664911906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,3072,0.022802132368087768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2560,0.021303466955820718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,2048,0.020153599977493285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1024,0.01606933375199636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,768,0.015386666854222616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,1536,0.01879253387451172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,512,0.015291733543078103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,256,0.0151146670182546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,128,0.014899200201034546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,64,0.014823466539382935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,32,0.014917332927385965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,7168,65536,0.17776106198628744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,8192,0.034833065668741864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,12288,0.04228693246841431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,10240,0.037257599830627444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,16384,0.05140053431193033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,4096,0.024268800020217897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,5120,0.026270933945973712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,7168,0.03144213358561198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,6144,0.028963200251261395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3584,0.022977066040039063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,3072,0.021780266364415487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2560,0.020716800292332967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,2048,0.01943146586418152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1536,0.018219733238220216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,1024,0.01602773368358612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,768,0.015316266814867655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,512,0.015415466825167336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,256,0.015046399831771851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,128,0.014685866236686707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,65536,0.15384853680928548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,64,0.014850133657455444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,6144,32,0.014954666296641031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,8192,0.02990506688753764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,10240,0.03340906699498494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,12288,0.03731946547826131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,16384,0.04400213162104289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,6144,0.02600319981575012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,5120,0.024231467644373575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,7168,0.028102399905522664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,4096,0.02231360077857971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3584,0.021546665827433267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,3072,0.02065066695213318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2560,0.01986666719118754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,2048,0.018755199511845906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1536,0.016548267006874083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,1024,0.015246933698654175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,512,0.014997333288192749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,768,0.015459199746449789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,65536,0.12723946571350098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,256,0.015122133493423461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,128,0.014889599879582724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,64,0.014773333072662353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,5120,32,0.014755200346310934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,12288,0.03511573473612468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,8192,0.02917120059331258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,16384,0.040982401371002196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,10240,0.03236266573270162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,7168,0.027479465802510577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,6144,0.026177066564559936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,5120,0.02281173268953959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,4096,0.021177599827448525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3584,0.020376533269882202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,3072,0.019732266664505005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2560,0.018989866971969603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,2048,0.017803732554117838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1536,0.015836800138155617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,1024,0.015498666961987814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,65536,0.10931413173675537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,768,0.015401599804560342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,512,0.015175466736157736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,256,0.014990933736165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,64,0.015053866306940713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,128,0.014702933033307395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,4096,32,0.01495039959748586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,8192,0.02767360011736552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,12288,0.033297065893809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,10240,0.030436267455418903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,16384,0.0382261315981547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,7168,0.026467200120290118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,6144,0.025111466646194458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,4096,0.020536533991495767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,5120,0.021686400969823202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3584,0.019820799430211387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2560,0.018668800592422485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,2048,0.01639893352985382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,3072,0.019324799378712974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,65536,0.09742080370585124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1536,0.0159061332543691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,1024,0.015541332960128783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,768,0.015491200486818948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,512,0.015165866414705912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,256,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,128,0.014672000209490457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,64,0.014759467045466105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3584,32,0.014879999558130899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,12288,0.03166933258374532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,10240,0.028859732548395793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,16384,0.036157866319020585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,8192,0.027180800835291546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,6144,0.024386133750279745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,5120,0.021064533789952596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,7168,0.025944532950719197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,4096,0.02001919945081075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3584,0.019246933857599895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2560,0.016964266697565712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,3072,0.01880426605542501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,65536,0.08822080294291178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,2048,0.01614293356736501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1536,0.015965867042541503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,1024,0.015638400117556253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,768,0.01530346671740214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,512,0.015080533425013223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,256,0.014917332927385965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,128,0.014776532848676046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,64,0.014825600385665893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,3072,32,0.014645333091417948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,16384,0.03291413386662801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,10240,0.026843732595443724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,12288,0.028706133365631104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,8192,0.024650667111078897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,7168,0.024139734109242757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,6144,0.02323733369509379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,5120,0.020456532637278237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,4096,0.01928960084915161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,65536,0.07517546812693278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3584,0.018706132968266807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,3072,0.0171615997950236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2560,0.016389333208402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,2048,0.016028799613316855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1536,0.015820800264676412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,768,0.015326933066050211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,1024,0.015523200233777365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,512,0.015195733308792115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,256,0.014874666929244995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,64,0.014739200472831726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,128,0.014782933394114175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2560,32,0.014730667074521383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,12288,0.02613226572672526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,10240,0.0247488001982371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,8192,0.023355732361475624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,16384,0.029330132404963176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,7168,0.022691200176874794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,6144,0.022030933698018392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,65536,0.06642026503880818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,5120,0.019698133071263633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,4096,0.018667733669281004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3584,0.01726400057474772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,3072,0.016693333784739174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2560,0.016246400276819863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,2048,0.016214399536450704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1536,0.015899733702341715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,1024,0.015421866377194723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,768,0.015287466843922935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,512,0.015150933464368185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,256,0.014921599626541137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,128,0.014661332964897156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,64,0.014670933286348978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,2048,32,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,12288,0.024132267634073893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,16384,0.026947200298309326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,10240,0.023370667298634847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,8192,0.022295467058817544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,65536,0.05442453225453695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,7168,0.021848533550898233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,6144,0.021345067024230956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,5120,0.018260266383488974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,4096,0.01755626598993937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3072,0.016785067319869996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,3584,0.017357865969340004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2560,0.016643200318018594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,2048,0.016247466206550598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1536,0.015929599603017174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,1024,0.015521066387494406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,768,0.015307733416557312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,256,0.01495680014292399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,128,0.014722133676211039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,512,0.015172266960144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,64,0.014567466576894126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1536,32,0.014710399508476257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,16384,0.023732266823450723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,12288,0.022061866521835328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,10240,0.02142613331476847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,8192,0.020823466777801513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,65536,0.0440170685450236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,6144,0.018870399395624796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,7168,0.019725867112477622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,5120,0.01797119975090027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,4096,0.017744000752766928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3584,0.017366399367650352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,3072,0.016886399189631144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2560,0.01644480029741923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1536,0.01586026648680369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,2048,0.0160863995552063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,1024,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,768,0.015224533279736838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,512,0.015086932977040609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,256,0.014937600493431092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,128,0.014551466703414917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,32,0.014678399761517844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,1024,64,0.01464959979057312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,16384,0.02279040018717448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,10240,0.02071573336919149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,8192,0.01889386574427287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,12288,0.02103786667188009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,65536,0.038423466682434085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,7168,0.01918506622314453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,6144,0.019106133778889974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,5120,0.018323200941085815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,4096,0.017434666554133095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3584,0.01713493267695109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2560,0.016445866227149962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,3072,0.016863999764124553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,2048,0.016216533382733662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1536,0.01592639982700348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,1024,0.01539306640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,768,0.015250133474667868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,256,0.014842666188875833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,512,0.015103999773661295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,128,0.014663466811180114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,64,0.014549332857131957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,16384,0.021422932545344033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,768,32,0.0146506667137146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,12288,0.01938026746114095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,10240,0.019401599963506065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,65536,0.03219626744588216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,8192,0.018948266903559365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,7168,0.019703465700149535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,4096,0.017618133624394735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,5120,0.018155733744303383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,6144,0.01914560000101725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3072,0.01713386575380961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,3584,0.017246933778127034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2560,0.016565333803494772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,2048,0.016270933548609416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1536,0.01586026648680369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,1024,0.01551253298918406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,768,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,512,0.015060266852378846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,256,0.014972800016403198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,128,0.014686933159828186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,64,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,512,32,0.014692266782124838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,65536,0.026096000274022417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,16384,0.01955946683883667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,12288,0.01914773384730021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,10240,0.019605332612991334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,8192,0.019037866592407228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,7168,0.01967039903004964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,6144,0.019205333789189656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,4096,0.017605332533518474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,5120,0.018263467152913413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3584,0.017173333962758382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,3072,0.016990933815638223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2560,0.01656426688035329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,2048,0.016135467092196147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1536,0.01569066643714905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,1024,0.015380266308784484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,768,0.015171200037002563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,512,0.014957867066065469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,256,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,64,0.014673067132631936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,128,0.014521599809328715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,256,32,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,65536,0.023113600413004556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,16384,0.019441066185633342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,12288,0.019041067361831664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,8192,0.018773333231608073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,10240,0.019052799542744955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,7168,0.01925440033276876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,6144,0.019002666076024376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,5120,0.018027732769648232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,4096,0.01733120083808899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3584,0.01696853240331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,3072,0.016540799538294473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2560,0.016354133685429893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,2048,0.016084266702334087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1536,0.0157642662525177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,768,0.015117866794268289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,512,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,1024,0.0151936004559199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,256,0.014786133170127868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,128,0.014556800325711569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,32,0.014565333724021912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,128,64,0.014516266187032065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,65536,0.021524266401926676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,16384,0.019307732582092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,12288,0.01880319913228353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,10240,0.019090133905410766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,8192,0.01858560045560201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,6144,0.018914133310317993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,5120,0.017845332622528076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,7168,0.019246933857599895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,4096,0.017288533846537273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3584,0.01686186591784159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,3072,0.016793600718180337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2560,0.01627840002377828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,2048,0.016009599963823954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1536,0.015768532951672874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,1024,0.015175466736157736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,768,0.015084800124168397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,512,0.014883200327555338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,256,0.014656000336011252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,128,0.014638933539390563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,64,0.014432000120480857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,64,32,0.014590932925542196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,65536,0.02173973321914673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,16384,0.01915839910507202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,12288,0.01882879932721456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,10240,0.019132800896962485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,8192,0.018686934312184652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,7168,0.019156267245610557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,5120,0.018029866615931193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,6144,0.018729599316914876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3072,0.01664959987004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,3584,0.01698453426361084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,4096,0.017234132687250773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2560,0.016246400276819863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,2048,0.015938133001327515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1536,0.015715199708938598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,1024,0.015251200397809347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,768,0.015145599842071533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,512,0.014922666549682616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,256,0.014697600404421488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,128,0.014640000462532044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,64,0.014604799946149192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,16,32,32,0.014597333470980325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,7168,0.1730079968770345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,8192,0.19575680096944173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,10240,0.2381312052408854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,12288,0.28471358617146814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,5120,0.12784852981567382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,6144,0.15033814112345378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,4096,0.10453973611195881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,16384,0.37457173665364585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3584,0.09714773495992025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2048,0.06050133307774862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,3072,0.08338027000427246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,2560,0.07179413636525472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1536,0.05597440004348755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,1024,0.038258135318756104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,64,0.0160778671503067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,128,0.018026665846506754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,256,0.020882133642832437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,512,0.02683839996655782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,768,0.033073065678278606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,65536,32,0.016461867094039916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,8192,0.058942933877309166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,10240,0.06969707012176514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,12288,0.08118613560994467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,16384,0.10283199946085612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,6144,0.04844799836476644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,7168,0.053743998209635414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,5120,0.04309013287226359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,4096,0.037393065293629964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2560,0.029377067089080812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3072,0.03224426706631978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,3584,0.0349237322807312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,2048,0.026070400079091387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1536,0.02325119972229004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,1024,0.02057173252105713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,768,0.01916159987449646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,512,0.017463467518488564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,256,0.015275733669598899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,128,0.015081600348154704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,64,0.015132799744606018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,32,0.015058133006095886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,10240,0.05657920042673746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,12288,0.06555200020472209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,16384,0.08175679842631021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,7168,0.044788265228271486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,8192,0.04845866759618123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,6144,0.040880000591278075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,16384,65536,0.35751253763834634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,4096,0.03222186764081319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,5120,0.03622613350550334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3584,0.030126933256785077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,3072,0.029747200012207032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2560,0.025729066133499144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,2048,0.023550933599472045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1536,0.0219050665696462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,768,0.01836693286895752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,512,0.015803733468055726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,1024,0.019299199183781944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,256,0.015401599804560342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,64,0.015261866648991904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,128,0.015147733688354491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,32,0.015068800250689188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,12288,65536,0.2798645337422689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,10240,0.05033599932988485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,12288,0.056876798470815025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,16384,0.07108799616495767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,8192,0.047704533735911055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,6144,0.03600000143051148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,7168,0.03979839881261189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,5120,0.03327253262201945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3072,0.025618133942286174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,4096,0.029270400603612263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2560,0.023916800816853843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,3584,0.027349332968393963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,2048,0.02209386626879374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1536,0.020275199413299562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,1024,0.018592000007629395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,768,0.016810667514801026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,512,0.015523200233777365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,256,0.01518186628818512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,128,0.014934399724006652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,64,0.014863999684651694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,32,0.015035733580589294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,10240,65536,0.23661866188049316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,12288,0.048793598016103105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,10240,0.042998401323954265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,16384,0.058764799435933435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,8192,0.042004267374674484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,7168,0.03472213347752889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,5120,0.029198932647705077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,6144,0.032022400697072344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3072,0.023307732741038003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,3584,0.024675200382868447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2560,0.021916800737380983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,4096,0.026268800099690754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,2048,0.020460800329844157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1536,0.01998293399810791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,1024,0.01759786605834961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,768,0.015065600474675497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,512,0.01520639955997467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,256,0.01511253317197164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,128,0.014817066987355552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,64,0.01480959951877594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,32,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,8192,65536,0.19236693382263184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,10240,0.042873601118723556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,8192,0.03823359807332356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,12288,0.04784746567408244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,16384,0.057595733801523843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,4096,0.024906667073567708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,7168,0.032551467418670654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,6144,0.03044053316116333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,5120,0.02765333255132039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3072,0.022722133000691733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,3584,0.023754666248957314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2560,0.0214026669661204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,2048,0.02004693349202474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1536,0.01895893414815267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,1024,0.01566933294137319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,768,0.01569919983545939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,512,0.015197867155075073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,256,0.015121066570281982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,128,0.014733866850535075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,64,0.014819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,65536,0.17456639607747396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,7168,32,0.014779733618100485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,8192,0.03270933429400126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,10240,0.03722560008366903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,12288,0.041272532939910886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,16384,0.04998079935709636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,5120,0.026014933983484905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,4096,0.024421334266662598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,6144,0.028886399666468304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,7168,0.030730666716893514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3584,0.022685867547988892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,3072,0.022004266579945884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2560,0.020705066124598184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,2048,0.019308799505233766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1536,0.01831679940223694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,1024,0.015438933173815408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,768,0.015466666221618653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,512,0.015237333377202353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,256,0.01504746675491333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,65536,0.15100266138712565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,128,0.014816000064214071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,32,0.014765866597493491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,6144,64,0.014781866470972696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,16384,0.04357226689656575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,8192,0.029157332579294842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,10240,0.03332479993502299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,12288,0.03667200009028117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,6144,0.025574400027592974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,5120,0.024141865968704223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,4096,0.02231360077857971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,7168,0.02773653268814087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3584,0.021125332514444987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2048,0.018739199638366698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,3072,0.02070186734199524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,2560,0.019733333587646486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,65536,0.13025813102722167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1536,0.017207467555999757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,1024,0.015701333681742348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,512,0.015153066317240397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,768,0.0152319997549057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,64,0.014734933773676554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,256,0.014984533190727234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,128,0.014598400394121806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,5120,32,0.014729600151379904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,10240,0.031982932488123575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,8192,0.026769065856933595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,12288,0.03482346534729004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,16384,0.039961600303649904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,6144,0.02366080085436503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,7168,0.024884267648061117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,5120,0.02229866584142049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,4096,0.020889600118001304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3584,0.020103466510772706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,3072,0.019451733430226645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2048,0.017816533644994102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,2560,0.018837332725524902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,65536,0.10629013379414876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1536,0.015638400117556253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,768,0.015192533532778421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,1024,0.015446399648984274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,512,0.015269333124160766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,256,0.014879999558130899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,64,0.01474666694800059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,128,0.014614400267601014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,4096,32,0.01474240024884542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,16384,0.03747626543045044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,8192,0.025171200434366863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,12288,0.032671999931335446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,10240,0.030237867434819536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,7168,0.023887999852498374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,5120,0.02158613403638204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,6144,0.02298133373260498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,4096,0.020478934049606323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3072,0.019269333283106486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,3584,0.01975253423055013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,65536,0.09648640155792236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2048,0.01705066760381063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,2560,0.018691200017929076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1536,0.016057599584261575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,1024,0.01552959978580475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,768,0.015390933553377787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,512,0.015191466609636942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,256,0.014884266257286071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,128,0.014749866724014283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,64,0.014698666334152222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3584,32,0.014800000190734863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,10240,0.02826559940973918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,8192,0.0264789342880249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,16384,0.03498560190200806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,12288,0.030572799841562907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,7168,0.02535039981206258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,4096,0.019773866732915243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,5120,0.02087679902712504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,6144,0.02412373423576355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3584,0.01934400002161662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,65536,0.08674240112304688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,3072,0.01872746745745341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2048,0.01581653356552124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,2560,0.01770240068435669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1536,0.015405866503715514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,768,0.015317333738009134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,1024,0.015636266271273295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,512,0.015172266960144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,256,0.014806399742762247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,128,0.014612266421318054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,64,0.01474346617857615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,3072,32,0.01474453310171763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,12288,0.028361600637435914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,10240,0.0265066663424174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,16384,0.03185173273086548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,8192,0.02483946681022644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,6144,0.023280000686645506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,7168,0.023859200874964397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,5120,0.02009920080502828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,4096,0.019236266613006592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,65536,0.07462080319722494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3584,0.018703999121983846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2560,0.016760534048080443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,3072,0.01731946667035421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,2048,0.01578879952430725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1536,0.01574399967988332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,1024,0.015544533729553223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,768,0.015241600076357522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,256,0.014780799547831217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,512,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,128,0.014647466937700906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,64,0.014731733004252115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2560,32,0.014694399634997048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,12288,0.026011733214060466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,10240,0.024756266673405965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,16384,0.028663466374079388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,8192,0.023195733626683555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,7168,0.02253119945526123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,6144,0.022027732928593953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,5120,0.019450666507085164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,65536,0.06428906520207724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,4096,0.018602667252222697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3584,0.017948800325393678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,3072,0.01649493376413981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2560,0.016284799575805663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,2048,0.016233600179354348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1536,0.015833600362141927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,1024,0.015505066514015198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,768,0.015413332978884378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,64,0.014711466431617738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,512,0.01506239970525106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,256,0.014919466773668923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,32,0.014804266889890037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,2048,128,0.01470080018043518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,10240,0.023105067014694215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,12288,0.024256000916163124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,8192,0.02211839954058329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,16384,0.025942399104436235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,7168,0.021655466159184775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,6144,0.021138133605321248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,5120,0.01912213365236918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,4096,0.01744106610616048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,65536,0.052552533149719236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3584,0.01713386575380961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,3072,0.016973867019017538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2560,0.01644266645113627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1536,0.015793066223462424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,2048,0.01607360045115153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,1024,0.015500799814860026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,256,0.014963199694951376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,512,0.015006933609644571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,768,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,128,0.014582399527231851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,64,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1536,32,0.014657066265741984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,16384,0.023772799968719484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,12288,0.02194026708602905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,10240,0.021579732497533165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,8192,0.020507733027140297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,65536,0.04185813268025716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,7168,0.01932800014813741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,5120,0.018286933501561485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,6144,0.01883413394292196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,4096,0.017425066232681273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3584,0.017103999853134155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,3072,0.016754132509231568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2560,0.01650879979133606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,2048,0.016119466225306193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1536,0.015794133146603904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,1024,0.015426133076349893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,768,0.015221333503723145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,512,0.015029333035151162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,256,0.01476693352063497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,128,0.014662399888038635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,64,0.01458453337351481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,1024,32,0.014632532993952433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,12288,0.021026132504145305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,16384,0.02211306691169739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,65536,0.0366431991259257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,10240,0.020193066199620566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,8192,0.0185589333375295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,7168,0.019127466281255088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,6144,0.018307199080785118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,5120,0.018281600872675576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,4096,0.017528533935546875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3584,0.0171509325504303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,3072,0.016833066940307617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2560,0.016334933042526246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,2048,0.01618880033493042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1536,0.015714133779207863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,768,0.015289599696795145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,1024,0.01544426679611206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,256,0.014876799782117209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,512,0.01492800017197927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,64,0.01469013293584188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,128,0.014542933305104574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,768,32,0.014626133441925048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,16384,0.020670932531356812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,12288,0.019595734278361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,65536,0.030554666121800737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,10240,0.01932800014813741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,8192,0.018770132462183634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,6144,0.018953599532445273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,7168,0.019232000907262167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,5120,0.018295466899871826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,4096,0.017491199572881064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3584,0.01713386575380961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,3072,0.016835200786590575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2048,0.016099199652671814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,2560,0.016375466187795003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1536,0.015756799777348836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,1024,0.015474133690198264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,768,0.015258666872978211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,512,0.015015467007954916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,256,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,128,0.01461120049158732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,64,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,512,32,0.014667733510335287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,65536,0.024740266799926757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,16384,0.0190720001856486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,10240,0.01949440042177836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,12288,0.018850133816401164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,8192,0.018987733125686645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,7168,0.019372800985972084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,6144,0.01871573328971863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,5120,0.0182751993338267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3584,0.01723946730295817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,4096,0.017436800400416057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,3072,0.016845866044362386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2560,0.01641813317934672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,2048,0.01618346671263377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1536,0.015758933623631795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,1024,0.015334399541219077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,768,0.015228799978892007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,256,0.014707199732462563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,512,0.014948266744613647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,128,0.01461120049158732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,64,0.014562132954597472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,256,32,0.014595199624697366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,65536,0.022039467096328737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,12288,0.018849066893259683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,16384,0.018966400623321535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,8192,0.018602667252222697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,10240,0.01909866730372111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,7168,0.019063466787338258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3584,0.01673813263575236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,5120,0.017802667617797852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,6144,0.018734933932622273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,4096,0.01728960076967875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,3072,0.016661333044370015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2560,0.016194132963816325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1024,0.015209600329399109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,2048,0.015898666779200234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,1536,0.015635200341542563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,768,0.015078399578730264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,512,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,256,0.01477013329664866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,128,0.014375467101732889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,64,0.014641066392262777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,128,32,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,65536,0.019857066869735717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,16384,0.018994132677714028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,10240,0.019074134031931558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,12288,0.01857066750526428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,8192,0.018488534291585288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,7168,0.01890666683514913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,6144,0.01885333259900411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,5120,0.01779306729634603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,4096,0.017143466075261436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3584,0.0168778657913208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,3072,0.01646293302377065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2560,0.016302933295567833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,2048,0.015983999768892924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1536,0.01546986699104309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,1024,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,768,0.01514240006605784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,512,0.014978133638699851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,256,0.014735999703407287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,128,0.014472533265749613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,64,0.014549332857131957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,64,32,0.014601600170135499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,65536,0.020147200425465903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,16384,0.01876266598701477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,12288,0.018492799997329713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,10240,0.018968532482782997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,8192,0.018408532937367758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,7168,0.018760534127553304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,6144,0.018760534127553304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,4096,0.0172437330087026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,5120,0.01764906644821167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3584,0.01687999963760376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,3072,0.01646933356920878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2560,0.016055466731389363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,2048,0.0160181333621343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1024,0.015109333395957946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,1536,0.01551466683546702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,768,0.014958932995796204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,512,0.014757333199183145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,256,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,128,0.01442026694615682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,64,0.014535466829935709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,8,32,32,0.014552533626556396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,7168,0.17158293724060059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,8192,0.19598719278971355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,10240,0.23917439778645835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,12288,0.2834837277730306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,6144,0.1498037338256836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,5120,0.12692373593648273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,4096,0.10415679613749187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,16384,0.3756437301635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3584,0.09644479751586914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1536,0.05564373334248861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2560,0.07329386870066325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,3072,0.0831157366434733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,1024,0.0380351980527242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,2048,0.06007999976476034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,256,0.020923733711242676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,128,0.020350933074951172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,64,0.01906879941622416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,512,0.026726400852203368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,768,0.032818132638931276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,65536,32,0.01902186671892802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,8192,0.05887146790822347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,10240,0.06944959958394369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,12288,0.08086293538411458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,16384,0.10262826283772786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,5120,0.04272426764170329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,7168,0.053657599290212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,6144,0.04812693198521932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,4096,0.03712213436762492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2560,0.029202133417129517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3072,0.03215786616007487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,3584,0.03455253442128499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,2048,0.026102399826049803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1024,0.020639999707539877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,1536,0.02335466742515564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,768,0.0191594660282135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,512,0.017534933487574258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,256,0.015496533115704855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,128,0.01508799990018209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,64,0.014938666423161825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,32,0.015019733707110086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,10240,0.05606506665547689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,12288,0.06511146624883016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,16384,0.08110933303833008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,8192,0.04798613389333089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,7168,0.0443829337755839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,6144,0.0403221329053243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,16384,65536,0.3568565368652344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,5120,0.03609386682510376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3584,0.029719465970993043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,4096,0.032181332508722946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,3072,0.029768532514572142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2560,0.025360000133514405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,2048,0.023362133900324503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1536,0.021585067113240562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,1024,0.019324799378712974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,512,0.01551253298918406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,768,0.018272000551223754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,256,0.015389866630236306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,128,0.015092266599337259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,64,0.015140266219774882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,32,0.015229866902033488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,12288,65536,0.2798410733540853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,10240,0.04943893353144328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,12288,0.05744533141454061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,16384,0.07056000232696533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,8192,0.04754773378372192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,6144,0.03637866576512654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,7168,0.0392576018969218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,5120,0.03279146750768026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3072,0.025260800123214723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,4096,0.029231999317804975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,3584,0.027357866366704304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2560,0.023727999130884806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,2048,0.02175146738688151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1536,0.0203274667263031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,1024,0.01854613423347473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,768,0.016426666577657064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,512,0.015146666765213012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,256,0.015217066804567973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,128,0.014923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,64,0.014991999665896098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,32,0.014873600006103516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,10240,65536,0.2357738653818766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,10240,0.04271573225657145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,16384,0.0584277351697286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,12288,0.04848106702168782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,5120,0.02926186720530192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,6144,0.03189866741498311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,8192,0.04179626703262329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,7168,0.03439146677652995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3584,0.024730666478474935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,3072,0.023441066344579063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2560,0.021947733561197915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,4096,0.026837333043416338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,2048,0.020456532637278237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1536,0.01931519905726115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,1024,0.01752426624298096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,768,0.015837867061297098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,512,0.01548906664053599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,256,0.015086932977040609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,128,0.0150218665599823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,64,0.014884266257286071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,32,0.014932266871134438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,8192,65536,0.190937598546346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,8192,0.03794133265813192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,10240,0.04236373504002889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,12288,0.04771946668624878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,16384,0.05667519966761271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,4096,0.02486293315887451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,5120,0.02757866581281026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,6144,0.0301962673664093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,7168,0.032281599442164105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3584,0.023785599072774253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,3072,0.022921599944432578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2048,0.020090667406717937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,2560,0.021331199010213218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1536,0.018833067019780478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,1024,0.01564800043900808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,768,0.01558080017566681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,512,0.01546346644560496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,256,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,128,0.01477120021979014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,64,0.014893866578737893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,32,0.014841600259145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,7168,65536,0.17287039756774902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,10240,0.03720320065816243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,12288,0.04147200187047322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,16384,0.049958399931589764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,8192,0.03543999989827474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,5120,0.0262773334980011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,4096,0.023838933308919272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,6144,0.02877013285954793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,7168,0.030687999725341798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3584,0.02291946609814962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,3072,0.021553067366282146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2560,0.020782933632532755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,2048,0.019326933224995933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1536,0.018281600872675576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,768,0.015185067057609558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,1024,0.015438933173815408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,512,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,256,0.015066666404406228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,65536,0.15155092875162762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,128,0.014892799655596414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,64,0.014732799927393594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,6144,32,0.014800000190734863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,8192,0.029370667537053426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,12288,0.03663040002187093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,10240,0.03296746611595154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,16384,0.043508267402648924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,6144,0.025756800174713136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,5120,0.024112000068028768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,4096,0.02217386762301127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,7168,0.027636265754699706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3584,0.021411200364430748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,3072,0.020614399512608846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2560,0.01946880022684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,2048,0.018716800212860107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1536,0.016436266899108886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,768,0.015343999862670899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,512,0.015285332997639975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,1024,0.01553600033124288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,65536,0.12474559942881266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,256,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,128,0.014683733383814493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,64,0.014819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,5120,32,0.014703999956448874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,10240,0.03182400067647298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,16384,0.039613866806030275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,8192,0.028587732712427778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,12288,0.03439893325169881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,7168,0.024936532974243163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,6144,0.023679999510447185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,5120,0.022454400857289634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,4096,0.020819199085235596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3584,0.020106667280197145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,3072,0.019421867529551187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2560,0.018997333447138467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,65536,0.10534079869588216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,2048,0.017847466468811034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1536,0.01597119967142741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,1024,0.01556373337904612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,768,0.01532586713631948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,512,0.015211733182271323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,256,0.014909866452217101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,128,0.0147189329067866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,64,0.01474453310171763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,4096,32,0.014727466305096946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,10240,0.03014933268229167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,12288,0.03237866759300232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,8192,0.024937599897384644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,16384,0.03742186625798543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,7168,0.024284799893697105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,6144,0.022674133380254112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,4096,0.020304000377655028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,5120,0.021413334210713706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3584,0.01983786622683207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,3072,0.019028266270955406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2560,0.018548266092936198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,2048,0.016639999548594155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,65536,0.09566613038380942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1536,0.015925332903862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,768,0.01532799998919169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,1024,0.015554133057594299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,512,0.015016532937685647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,256,0.014842666188875833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,128,0.014775466918945313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,32,0.014789332946141561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3584,64,0.014712533354759217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,10240,0.02799573342005412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,8192,0.026549333333969118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,16384,0.03470826546351115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,12288,0.030356266101201373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,7168,0.025224532683690386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,6144,0.024139734109242757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,5120,0.020843732357025146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,4096,0.019730132818222047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3584,0.019027199347813925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,65536,0.08228586514790853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,3072,0.018659200270970663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2560,0.016926934321721397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,2048,0.01636799971262614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1536,0.015997866789499916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,1024,0.015517866611480713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,768,0.015188266833623251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,256,0.014829867084821067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,512,0.015069866180419922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,128,0.014756266276041666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,64,0.014681599537531533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,3072,32,0.014693333705266317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,12288,0.028191999594370527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,10240,0.026473599672317504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,16384,0.03178453246752421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,8192,0.024421334266662598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,7168,0.023871999979019166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,6144,0.023042132457097374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,5120,0.020233599344889323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,4096,0.019049600760142008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,65536,0.07267306645711263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3584,0.01869973341623942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,3072,0.017921066284179686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2560,0.01669973333676656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,2048,0.01614293356736501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1536,0.01595200002193451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,1024,0.015485866864522298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,768,0.015264000495274863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,512,0.015030399958292643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,128,0.014808533589045205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,64,0.014734933773676554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,256,0.014750933647155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2560,32,0.014722133676211039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,12288,0.02579413255055745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,10240,0.02482346693674723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,16384,0.02842346628506978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,8192,0.02328426639238993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,7168,0.02245546579360962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,6144,0.021972266832987468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,5120,0.019578667481740315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,4096,0.01859626571337382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,65536,0.06300373474756876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3584,0.017322667439778647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,3072,0.016544000307718913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2560,0.016458666324615477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,2048,0.0160181333621343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1024,0.01548479994138082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,1536,0.01589120030403137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,768,0.015201066931088766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,512,0.015118933717409768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,128,0.014678399761517844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,256,0.014637866616249084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,64,0.014757333199183145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,2048,32,0.01463573376337687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,16384,0.02604373296101888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,12288,0.023869866132736207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,10240,0.023264000813166298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,8192,0.0224671999613444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,7168,0.021373866001764934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,5120,0.0192138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,6144,0.020871466398239134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,65536,0.051345066229502356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,4096,0.017220266660054526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3584,0.016613333423932394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2560,0.016451199849446617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,3072,0.01673706571261088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,2048,0.016220800081888833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1024,0.015464533368746439
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,1536,0.01580586632092794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,768,0.015194666385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,512,0.014963199694951376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,256,0.015029333035151162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,128,0.014728533228238425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,64,0.014626133441925048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1536,32,0.014672000209490457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,16384,0.023337600628534953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,12288,0.02174506584803263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,10240,0.021196800470352172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,8192,0.02057066758473714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,65536,0.0406826655069987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,7168,0.019259732961654664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,6144,0.018553600708643595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,5120,0.01758613387743632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,4096,0.01725119948387146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3072,0.016771199305852254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2560,0.016461867094039916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,3584,0.01710933248202006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,2048,0.01609173317750295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1536,0.01577173372109731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,1024,0.01539520025253296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,768,0.01511360009511312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,512,0.014965333541234336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,256,0.014630400141080222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,128,0.014923733472824097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,32,0.014974932869275412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,1024,64,0.015033599734306336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,16384,0.02220266660054525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,10240,0.02036799987157186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,8192,0.01845653255780538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,12288,0.020925867557525634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,65536,0.035634132226308186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,6144,0.019037866592407228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,5120,0.01797653237978617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,7168,0.01914773384730021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3584,0.01704746683438619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,4096,0.017429333925247193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,3072,0.016925867398579916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2560,0.01646933356920878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1536,0.015782399972279867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,2048,0.016195199886957803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,1024,0.015422933300336204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,768,0.015416533748308817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,512,0.015178666512171427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,256,0.01479039986928304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,128,0.014845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,64,0.014750933647155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,768,32,0.01476800044377645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,16384,0.02076479991277059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,65536,0.02964266737302144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,12288,0.01849386692047119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,10240,0.01916159987449646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,8192,0.018739199638366698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,7168,0.01932906707127889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,6144,0.018956800301869713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,5120,0.01830186645189921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,4096,0.017375999689102174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3584,0.017097600301106772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,3072,0.01690346598625183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2560,0.016710400581359863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1024,0.015516799688339234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,2048,0.016124799847602844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,1536,0.015749333302179973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,768,0.01530346671740214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,512,0.015190399686495461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,256,0.014867200454076131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,128,0.01470080018043518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,32,0.014842666188875833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,512,64,0.01458560029665629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,16384,0.01888213356335958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,65536,0.02458666761716207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,10240,0.019268266359965005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,12288,0.01880106727282206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,8192,0.018758400281270345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,7168,0.01906879941622416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,6144,0.018969599405924478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,5120,0.018142932653427125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,4096,0.017378133535385133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3584,0.017011199394861856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,3072,0.016774400075276693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2560,0.016382933656374613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,2048,0.016102400422096253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1024,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,1536,0.015701333681742348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,768,0.015119999647140503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,512,0.015006933609644571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,256,0.014708266655604044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,128,0.014663466811180114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,64,0.014643200238545737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,256,32,0.014665599664052328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,16384,0.018921599785486857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,65536,0.021425066391626994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,8192,0.01856000026067098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,12288,0.018627200524012247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,10240,0.019054933389027914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,7168,0.018965333700180054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,6144,0.018576000134150186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,5120,0.017879466215769448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,4096,0.017052799463272095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3584,0.01665279964605967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,3072,0.016562133034070333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2560,0.01619733373324076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,2048,0.015960533420244852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1536,0.015754666924476624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,1024,0.015335466464360556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,512,0.014995200435320535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,768,0.015059199929237366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,256,0.014750933647155762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,32,0.014618666966756186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,64,0.014629333217938741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,128,128,0.014654933412869772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,65536,0.01946880022684733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,16384,0.01864746610323588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,12288,0.018454400698343913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,10240,0.018722132841746012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,8192,0.018518400192260743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,7168,0.018873600165049235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,6144,0.0184714674949646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,5120,0.017618133624394735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2560,0.016008533040682473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3072,0.016390400131543477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,3584,0.01672640045483907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,4096,0.017100799083709716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,2048,0.015714133779207863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1536,0.015498666961987814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,1024,0.015237333377202353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,768,0.014939733346303306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,512,0.014991999665896098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,256,0.014823466539382935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,128,0.014657066265741984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,64,0.014737066626548768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,16384,0.018716800212860107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,64,32,0.014485333363215128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,65536,0.01948053240776062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,12288,0.01858560045560201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,8192,0.018589866161346436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,10240,0.018809600671132406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,7168,0.018796799580256145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,6144,0.018407466014226277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,5120,0.017678932348887125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,4096,0.017163733641306557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3072,0.016356266538302102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,3584,0.016897066434224447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2560,0.01594986617565155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,2048,0.015821866194407144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1536,0.015410133202870688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,1024,0.014998400211334228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,256,0.014637866616249084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,768,0.015027200182278952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,512,0.014794666568438211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,128,0.014563199877738953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,32,0.014599466323852539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,4,32,64,0.014593066771825156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,7168,0.17134720484415691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,8192,0.19522132873535156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,10240,0.23685866991678872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,12288,0.2844127972920736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,6144,0.1491541385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,5120,0.12694719632466633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,4096,0.10341440041859944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,16384,0.37366399765014646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3584,0.09340693155924479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2048,0.059876267115275064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,2560,0.07117439905802408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,3072,0.08264106909434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1536,0.05563626686731974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,1024,0.03781333367029826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,256,0.020924800634384157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,128,0.02020053267478943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,64,0.019002666076024376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,768,0.032595199346542356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,512,0.026537599166234334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,65536,32,0.01912320057551066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,8192,0.058619733651479086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,10240,0.0694602648417155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,12288,0.08035840193430582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,16384,0.10237867037455242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,5120,0.04256213506062825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,6144,0.047836800416310624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,7168,0.05335359970728556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,4096,0.036977068583170576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2560,0.029151999950408937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3072,0.03192639946937561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,3584,0.03456426858901977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,2048,0.02605866591135661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1024,0.020526933670043945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,768,0.01922453244527181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,1536,0.02329279979070028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,512,0.017382399241129557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,256,0.015334399541219077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,128,0.015096533298492431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,64,0.015065600474675497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,32,0.015020799636840821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,10240,0.05614613294601441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,12288,0.06506346861521403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,16384,0.081931734085083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,8192,0.04794453382492066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,7168,0.04376213153203328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,6144,0.04032320181528727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,16384,65536,0.3561823844909668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,4096,0.03174399932225545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3584,0.02990613381067912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,5120,0.0358570655186971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,3072,0.02977919975916545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2560,0.02518933415412903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,2048,0.023476266860961915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1536,0.02143146594365438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,1024,0.01923519968986511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,512,0.015321600437164306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,768,0.018183465798695883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,256,0.015528532862663268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,64,0.015256533026695251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,128,0.015149866541226705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,32,0.015122133493423461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,12288,65536,0.27770773569742835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,12288,0.05688213507334391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,10240,0.04984213511149089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,16384,0.0698527971903483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,8192,0.047465598583221434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,6144,0.03624746799468994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,7168,0.0394976019859314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,5120,0.03266879916191101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2560,0.023727999130884806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3072,0.025319466988245647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,4096,0.029020800193150835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,3584,0.02736213405927022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,2048,0.021924267212549843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1024,0.0185205340385437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,768,0.016733866930007935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,1536,0.0201749324798584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,512,0.015389866630236306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,128,0.01495573321978251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,256,0.015170133113861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,64,0.01499626636505127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,32,0.01493119994799296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,10240,65536,0.22716053326924643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,10240,0.042421332995096844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,12288,0.048121599356333415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,16384,0.05810346603393555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,8192,0.04143786827723185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,7168,0.03415573438008626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,5120,0.02895893255869548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,6144,0.03187733292579651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,4096,0.025969066222508747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3584,0.02478613257408142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,3072,0.023299199342727662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2560,0.022075732549031578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,2048,0.020477867126464842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1536,0.01924053430557251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,1024,0.017338667313257852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,768,0.01564906636873881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,512,0.015270400047302245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,256,0.015256533026695251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,128,0.014935466647148132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,64,0.014858667055765787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,32,0.01474666694800059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,8192,65536,0.18986666997273763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,16384,0.05612266858418783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,8192,0.038192001978556316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,12288,0.047337599595387775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,10240,0.04173760016759236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,4096,0.024771199623743693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,6144,0.030015999078750612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,5120,0.027510400613149005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,7168,0.03236160079638163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3584,0.023766400416692098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2560,0.021258666117986044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,2048,0.019937066237131755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,3072,0.022665599981943764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1536,0.018630399306615194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,1024,0.015833600362141927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,768,0.015356799960136414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,512,0.01518186628818512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,256,0.015180800358454385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,128,0.014807466665903726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,64,0.01488746702671051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,32,0.014728533228238425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,7168,65536,0.17418773969014484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,8192,0.03433813254038493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,12288,0.041324798266092935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,10240,0.037009068330128986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,16384,0.04972053368886312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,5120,0.026228266954421996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,4096,0.02414399981498718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,7168,0.031016532580057782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,6144,0.028386133909225463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3072,0.02172586719195048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,3584,0.022827732563018798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2560,0.02068693240483602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,2048,0.019397334257761637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1536,0.01826453407605489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,768,0.015520000457763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,512,0.015281066298484802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,1024,0.015412267049153647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,256,0.01513706644376119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,128,0.014749866724014283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,64,0.014798933267593383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,32,0.014758400122324624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,6144,65536,0.1505941390991211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,8192,0.029333333174387615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,10240,0.03283093372980754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,12288,0.03630613485972087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,16384,0.04264533519744873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,5120,0.02404159903526306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,6144,0.025707733631134034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,4096,0.022188800573349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,7168,0.02735146681467692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3584,0.021370667219161987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2560,0.019658666849136353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,3072,0.020619734128316244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,2048,0.018691200017929076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1536,0.01710933248202006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,1024,0.01572053333123525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,768,0.015433599551518759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,512,0.015052800377209982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,65536,0.12234559853871663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,256,0.01493333379427592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,128,0.014760532975196838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,32,0.014646400014559427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,5120,64,0.014734933773676554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,12288,0.03421226739883423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,16384,0.03979946772257487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,10240,0.031676799058914185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,8192,0.02868480086326599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,7168,0.025111466646194458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,6144,0.02367573380470276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,4096,0.02094506621360779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,5120,0.022243199745814006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3584,0.020450133085250854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,3072,0.01952426632245382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2560,0.018972800175348917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,2048,0.016768000523249307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,65536,0.10476906299591064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1536,0.015622400244077048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,1024,0.015641599893569946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,768,0.015201066931088766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,512,0.01503679951032003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,128,0.014856533209482829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,64,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,256,0.014995200435320535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,4096,32,0.014586666226387024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,12288,0.032103466987609866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,8192,0.024977066119511924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,10240,0.029908267656962077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,16384,0.036969598134358725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,7168,0.023797333240509033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,6144,0.022552533944447836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,4096,0.020332799355189005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,5120,0.021452800432840983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3584,0.019686400890350342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2560,0.018484266599019368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,3072,0.01908906698226929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,2048,0.01735360026359558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,65536,0.09353600343068441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1536,0.0159061332543691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,1024,0.015428266922632852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,768,0.015319466590881348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,512,0.015103999773661295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,256,0.014806399742762247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,128,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,64,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3584,32,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,10240,0.02793813347816467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,16384,0.03432533343633016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,8192,0.02681386669476827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,12288,0.029972267150878907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,7168,0.025278933842976886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,5120,0.020999467372894286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,6144,0.02402879993120829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,4096,0.019646932681401573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3584,0.01918399930000305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,65536,0.08336533705393473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,3072,0.018605866034825645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2048,0.01604586640993754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,2560,0.01769599914550781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1536,0.015894400080045064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,768,0.015273599823315939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,512,0.015129599968592325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,1024,0.01548906664053599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,256,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,128,0.014703999956448874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,64,0.01472106675306956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,3072,32,0.01478506624698639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,12288,0.02802133361498515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,16384,0.03179733355840047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,10240,0.026419200499852497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,8192,0.024486400683720908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,7168,0.023779199520746867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,4096,0.019053866465886436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,6144,0.022772266467412313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,5120,0.020337067047754922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3584,0.018870399395624796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,65536,0.0717024008433024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,3072,0.017930666605631508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2560,0.01665173371632894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,2048,0.01620586713155111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1536,0.01588266690572103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,1024,0.015436800320943198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,768,0.015266133348147073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,128,0.014798933267593383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,256,0.014689067006111145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,512,0.014906666676203408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,64,0.014653866489728292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2560,32,0.014732799927393594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,10240,0.024618667364120484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,12288,0.02553279995918274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,8192,0.023201066255569457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,16384,0.028277333577473956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,7168,0.02253440022468567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,6144,0.02173653244972229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,5120,0.01949333349863688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,65536,0.06191466649373373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,4096,0.0184063990910848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3584,0.01814613342285156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2560,0.016539733608563742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,3072,0.01664746701717377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,2048,0.01609173317750295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1536,0.015629866719245912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,768,0.015160533785820007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,1024,0.015436800320943198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,512,0.014879999558130899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,256,0.014946132898330688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,128,0.014590932925542196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,64,0.014615466197331747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,2048,32,0.014727466305096946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,10240,0.022897066672643025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,16384,0.026040534178415935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,12288,0.02396906614303589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,8192,0.022024534145991006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,7168,0.021310933430989585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,65536,0.050614400704701745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,6144,0.020786132415135702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,5120,0.018668800592422485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,4096,0.017248000701268515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3584,0.016953599452972413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,3072,0.016521599888801575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2048,0.016037333011627197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,2560,0.01620586713155111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1536,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,1024,0.015382400155067444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,768,0.014978133638699851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,512,0.014949333667755128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,256,0.01479573349157969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,128,0.014634666840235391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,64,0.014697600404421488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1536,32,0.014640000462532044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,16384,0.023283199469248454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,10240,0.0212991992632548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,12288,0.021805866559346517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,8192,0.020363734165827433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,65536,0.040194133917490646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,7168,0.019959467649459838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,6144,0.018463999032974243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,4096,0.01738986571629842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,5120,0.018692266941070557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3072,0.016683733463287352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,3584,0.017076265811920167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2560,0.016430933276812235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,2048,0.01613866686820984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1024,0.015453867117563882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,1536,0.015587199727694193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,768,0.015105066696802774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,512,0.015000533064206442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,128,0.015058133006095886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,256,0.014839466412862143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,64,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,1024,32,0.014954666296641031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,16384,0.022218666474024453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,12288,0.020593067010243736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,10240,0.0202239990234375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,8192,0.01920106609662374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,65536,0.0346346656481425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,7168,0.019107200702031455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,6144,0.018862932920455933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,5120,0.01805973251660665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,4096,0.017382399241129557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3584,0.017077332735061644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,3072,0.016827734311421712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2560,0.01630506714185079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,2048,0.016057599584261575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1024,0.015402666727701821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,1536,0.015556266903877259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,768,0.015457066893577575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,512,0.015034666657447815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,256,0.014965333541234336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,128,0.01477120021979014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,64,0.0146506667137146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,768,32,0.01474240024884542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,16384,0.02069759964942932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,12288,0.018710400660832724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,65536,0.028915200630823774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,10240,0.018988800048828126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,8192,0.018531199296315512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,7168,0.019232000907262167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,6144,0.018809600671132406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,5120,0.018075732390085857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3584,0.017164800564448038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,4096,0.017359999815622966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,3072,0.016940800348917644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2560,0.016574933131535848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,2048,0.016151466965675355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1536,0.01576533317565918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,1024,0.015424000223477683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,768,0.015187199910481772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,512,0.015098667144775391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,256,0.014857600132624308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,128,0.014692266782124838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,64,0.014689067006111145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,512,32,0.01467626690864563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,65536,0.023638399442036946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,16384,0.018895999590555827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,12288,0.018679465850194296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,10240,0.018935465812683107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,8192,0.018639999628067016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,7168,0.019010132551193236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,5120,0.018158932526906334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,6144,0.018689066171646118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3584,0.017121066649754844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,3072,0.01673706571261088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,4096,0.01746986707051595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2560,0.01639359990755717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,2048,0.016176000237464905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1536,0.015634133418401083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,1024,0.015431466698646545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,768,0.01511360009511312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,512,0.01497066617012024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,256,0.014868266383806863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,128,0.014625066518783569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,64,0.014705066879590353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,256,32,0.014605866869290671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,65536,0.021157334248224892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,12288,0.01841493248939514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,16384,0.018821332852045694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,10240,0.018703999121983846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,7168,0.018866133689880372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,8192,0.018367999792099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,6144,0.018425599733988444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,4096,0.017058134078979492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,5120,0.017603200674057008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3584,0.016872533162434897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,3072,0.016458666324615477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2560,0.016029866536458333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,2048,0.015863466262817382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1536,0.01567039986451467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,1024,0.015361066659291586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,768,0.015072000026702882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,512,0.014929067095120749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,256,0.014759467045466105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,128,0.014601600170135499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,64,0.014612266421318054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,65536,0.019339734315872194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,128,32,0.01458026667435964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,16384,0.018694400787353516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,12288,0.018337066968282065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,10240,0.018662399053573607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,8192,0.01847040057182312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,6144,0.01853546698888143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,5120,0.01745599905649821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,7168,0.018657066424687705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,4096,0.01689280072848002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3584,0.016838399569193523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,3072,0.01641813317934672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2560,0.01599679986635844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,2048,0.015622400244077048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1536,0.015429332852363586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,1024,0.01527253290017446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,768,0.014990933736165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,512,0.014834133783976236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,128,0.01469013293584188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,256,0.014647466937700906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,64,0.014703999956448874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,64,32,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,65536,0.018875734011332194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,16384,0.01856106718381246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,12288,0.018186666568120322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,8192,0.018412800629933675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,10240,0.018634666999181114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,7168,0.01858133276303609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,6144,0.018592000007629395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,5120,0.017385600010553996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,4096,0.016977065801620485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3072,0.016239999731381734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,3584,0.016682666540145875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2560,0.016064000129699708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,2048,0.015668267011642457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1536,0.015452800194422403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,1024,0.015145599842071533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,768,0.0150026669104894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,512,0.014826666315396628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,128,0.014595199624697366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,256,0.014659200112024942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,64,0.014547200004259745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,2,32,32,0.014562132954597472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,7168,0.17125120162963867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,8192,0.19476693471272785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,10240,0.23724586168924966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,12288,0.28222719828287757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,5120,0.1263423999150594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,6144,0.14889386494954426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,4096,0.10314239660898845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,16384,0.37742506663004555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3584,0.09610559940338134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2048,0.059544531504313145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1536,0.05579839944839478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,2560,0.07112533251444499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,3072,0.08255253632863363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,1024,0.03769386609395345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,256,0.020941867431004842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,128,0.02034986615180969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,512,0.026579199234644572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,768,0.0324181338151296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,64,0.019077332814534505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,65536,32,0.018987733125686645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,10240,0.06914133230845133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,8192,0.058317867914835606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,12288,0.08050026893615722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,16384,0.10114773114522298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,6144,0.04786773522694905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,5120,0.04244053363800049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,7168,0.053103999296824134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,4096,0.03701866865158081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2560,0.028819199403127032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3072,0.03194239934285482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,3584,0.03451306819915771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,2048,0.02606186668078105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1536,0.023203200101852416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,768,0.01928106745084127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,1024,0.020473599433898926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,512,0.017578667402267455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,256,0.015219199657440185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,128,0.01520853340625763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,32,0.015081600348154704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,64,0.015064533551534018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,10240,0.05629866520563761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,12288,0.06546133359273275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,16384,0.08296746412913004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,8192,0.04805546601613363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,7168,0.044078934192657473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,6144,0.04027626514434814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,16384,65536,0.3604480107625326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,4096,0.03179200092951457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,5120,0.03575253486633301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3584,0.029739733537038165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2560,0.025534933805465697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,3072,0.029666133721669513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,2048,0.023332266012827556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1536,0.02146773338317871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,512,0.015482667088508605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,1024,0.01932906707127889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,768,0.01810773412386576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,256,0.01542080044746399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,128,0.015175466736157736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,64,0.015133866667747497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,32,0.015061333775520325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,12288,65536,0.2769322713216146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,10240,0.048845867315928146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,12288,0.05710613330205282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,16384,0.06981653372446696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,8192,0.04752853314081828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,6144,0.036109864711761475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,7168,0.03869013388951619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,5120,0.03276906609535217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3072,0.025126399596532185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,4096,0.02895039916038513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2560,0.023622399568557738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,3584,0.02723199923833211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1536,0.020194133122762047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,768,0.01687893271446228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,2048,0.021758933862050377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,1024,0.018557866414388023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,512,0.015170133113861084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,256,0.015343999862670899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,128,0.01490239997704824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,64,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,32,0.014942933122316995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,10240,65536,0.23203412691752115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,12288,0.04824320077896118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,10240,0.04236266613006592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,16384,0.05813119808832804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,8192,0.041840000947316484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,7168,0.03423146804173787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,6144,0.03169813354810079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,5120,0.028905600309371948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,4096,0.025846399863560993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3072,0.02325013279914856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,3584,0.02466986576716105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2560,0.02202026645342509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1536,0.019156267245610557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,2048,0.02039253314336141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,1024,0.017568000157674155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,768,0.015685333808263143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,512,0.015426133076349893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,128,0.014719999829928079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,64,0.014978133638699851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,256,0.01495253344376882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,32,0.014845866958300272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,8192,65536,0.18976747194925944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,12288,0.046800001462300615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,16384,0.05565333366394043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,8192,0.03509973287582398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,10240,0.0418720006942749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,4096,0.024954666694005333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,7168,0.03223573366800944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,5120,0.027581866582234698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,6144,0.029890133937199907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3584,0.023718400796254476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,3072,0.022638932863871256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2560,0.021267199516296388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,2048,0.019883733987808228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1536,0.018705066045125326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,1024,0.01667413314183553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,768,0.015317333738009134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,256,0.015101866920789084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,512,0.015355733036994935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,128,0.014846932888031007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,64,0.014900267124176025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,65536,0.1718549410502116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,7168,32,0.014819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,8192,0.032763733466466265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,10240,0.038014932473500566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,16384,0.04938240051269531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,12288,0.041178667545318605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,4096,0.02406719923019409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,5120,0.026417066653569538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,7168,0.030580266316731768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,6144,0.02834239999453227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3584,0.02295253276824951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,3072,0.0216757337252299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2560,0.02056106726328532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,2048,0.01928106745084127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1536,0.018232532342274985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,512,0.015250133474667868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,1024,0.015763200322786965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,768,0.015575466553370157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,256,0.015065600474675497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,65536,0.14947199821472168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,128,0.014788267016410828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,32,0.014769066373507181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,6144,64,0.014773333072662353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,8192,0.029158399502436323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,10240,0.032737066348393754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,12288,0.03626453479131063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,16384,0.042625065644582114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,6144,0.0257205327351888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,5120,0.023950932423273723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,7168,0.027527467409769697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,4096,0.021967999140421548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3584,0.021266132593154907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,3072,0.020553600788116456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2048,0.018528000513712565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,2560,0.019553067286809285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1024,0.01530880033969879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,1536,0.017106133699417114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,65536,0.12643307050069172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,768,0.015171200037002563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,512,0.015185067057609558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,128,0.014774399995803832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,256,0.014971733093261719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,64,0.014824533462524414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,5120,32,0.014681599537531533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,12288,0.034381866455078125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,16384,0.039469865957895915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,8192,0.028509867191314698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,10240,0.031701334317525226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,6144,0.023735467592875162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,7168,0.025059199333190917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,5120,0.022155733903249104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,4096,0.02109866738319397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3584,0.020032000541687012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,3072,0.019579732418060304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2560,0.01889280080795288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,2048,0.017882666985193887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,65536,0.10393599669138591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1536,0.016391467054684958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,1024,0.015221333503723145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,768,0.015396266182263692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,512,0.015281066298484802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,128,0.014723199605941772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,256,0.01490239997704824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,32,0.014819199840227762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,4096,64,0.014565333724021912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,8192,0.02500159939130147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,16384,0.0366922656695048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,10240,0.029693865776062013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,12288,0.0319701333840688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,7168,0.023752532402674355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,6144,0.022738132874170938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,5120,0.021474132935206093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,4096,0.020360533396402994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3584,0.019671465953191122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,3072,0.019115734100341796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2560,0.018329600493113198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,65536,0.09400320053100586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,2048,0.016505600015322367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1536,0.015319466590881348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,1024,0.015553067127863566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,768,0.015226667126019796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,512,0.015320533514022827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,256,0.015001599987347921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,64,0.01476800044377645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,128,0.014709333578745524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3584,32,0.014747732877731323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,10240,0.02797226707140605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,16384,0.03442560036977132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,8192,0.026408533255259197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,12288,0.02972053289413452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,7168,0.02521173357963562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,6144,0.02424853245417277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,5120,0.020702934265136717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,4096,0.019612799088160195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3584,0.019076265891393027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,65536,0.08315412998199463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,3072,0.018634666999181114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2560,0.017497599124908447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,2048,0.016101333498954772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1536,0.015871999661127727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,1024,0.015619200468063355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,512,0.015148799618085226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,256,0.014860799908638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,768,0.015299200018246969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,128,0.014672000209490457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,64,0.014763733744621277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,3072,32,0.014780799547831217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,16384,0.03141546646753947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,12288,0.027983999252319335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,10240,0.026284799973169966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,8192,0.024264534314473472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,7168,0.023749333620071412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,6144,0.022932267189025878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,5120,0.02017813324928284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,4096,0.019074134031931558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,65536,0.07121280034383139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3584,0.018601600329081217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2560,0.0160970667997996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,3072,0.01726079980532328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1536,0.015943466623624166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,2048,0.01611733337243398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,1024,0.01539520025253296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,768,0.015255467096964518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,512,0.015003732840220132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,256,0.014942933122316995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,128,0.014762666821479798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,64,0.014668800433476768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2560,32,0.014706133802731832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,12288,0.02558079957962036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,16384,0.028332799673080444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,8192,0.0232586661974589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,10240,0.024537599086761473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,7168,0.0225600004196167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,5120,0.019666133324305217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,6144,0.02182719906171163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,65536,0.06144853432973226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,4096,0.018604799111684164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3584,0.017541333039601644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,3072,0.016595199704170227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2560,0.015863466262817382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,2048,0.015827199816703795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1536,0.015689599514007568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,1024,0.015247999628384908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,768,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,512,0.015042133132616677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,256,0.014727466305096946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,128,0.014725333452224732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,32,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,2048,64,0.014638933539390563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,16384,0.025998934110005693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,10240,0.022949333985646567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,12288,0.024114133914311726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,8192,0.02183893322944641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,65536,0.05025920073191324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,6144,0.020829866329828896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,7168,0.02153279980023702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,5120,0.01813439925511678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,4096,0.017077332735061644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3584,0.01694719990094503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,3072,0.016747732957204185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2048,0.01611840029557546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,2560,0.016264533003171287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1536,0.01551040013631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,1024,0.015362133582433065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,768,0.015140266219774882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,512,0.014964266618092855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,128,0.01463573376337687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,256,0.014761599898338317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,64,0.01470080018043518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1536,32,0.014694399634997048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,16384,0.023005867004394533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,12288,0.021618133783340453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,10240,0.021150932709376017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,8192,0.020171733697255452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,65536,0.03967466751734416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,7168,0.019603200753529868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,5120,0.01811520059903463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,6144,0.01848533352216085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,4096,0.017335466543833413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3584,0.01696959932645162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,3072,0.0165994664033254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2560,0.016416000326474507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,2048,0.016014933586120605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1536,0.015633066495259605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,1024,0.015318399667739869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,768,0.015107199549674988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,512,0.015015467007954916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,128,0.014935466647148132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,256,0.01477226714293162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,64,0.014891733725865683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,1024,32,0.015017599860827128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,16384,0.022094933191935222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,12288,0.02068693240483602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,10240,0.020014933745066323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,65536,0.03460906744003296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,8192,0.018990933895111084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,7168,0.019105066855748497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,6144,0.018476800123850504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,5120,0.017909334103266398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,4096,0.017454934120178223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3584,0.01699413259824117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,3072,0.01676266590754191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2048,0.01599360009034475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,2560,0.01618026693662008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1536,0.015629866719245912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,1024,0.01530239979426066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,768,0.01553813318411509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,512,0.015121066570281982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,128,0.014754133423169455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,256,0.014781866470972696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,64,0.014699733257293702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,768,32,0.014842666188875833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,16384,0.020786132415135702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,12288,0.01877440015474955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,65536,0.02903466622034709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,10240,0.018943999210993448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,8192,0.018577067057291667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,7168,0.019191465775171914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,6144,0.018691200017929076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,5120,0.018234666188557944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,4096,0.01745599905649821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3072,0.016804265975952148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,3584,0.017086933056513466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2048,0.016220800081888833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,2560,0.016343466440836587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1536,0.01570453345775604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,1024,0.015440000096956888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,768,0.015212800105412802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,512,0.015147733688354491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,128,0.014702933033307395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,256,0.014813866217931113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,64,0.014682666460673014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,512,32,0.014729600151379904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,65536,0.023976532618204753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,16384,0.018683733542760213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,10240,0.01889386574427287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,12288,0.018503467241923012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,8192,0.01850773294766744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,7168,0.019052799542744955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,6144,0.018593066930770875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,5120,0.018092799186706542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,4096,0.017352533340454102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3584,0.01697173317273458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,3072,0.016673066218694053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2560,0.016239999731381734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,2048,0.0160671999057134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1536,0.015585066874821982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,1024,0.015310933192571005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,768,0.015145599842071533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,512,0.014962133765220643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,256,0.014788267016410828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,128,0.01455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,32,0.014573867122332254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,256,64,0.014613333344459533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,16384,0.018809600671132406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,65536,0.02072426676750183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,12288,0.01841813325881958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,10240,0.018685867389043175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,7168,0.018835200866063436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,6144,0.018757333358128868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,8192,0.018205867211023966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,4096,0.016979199647903443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,5120,0.017542399962743125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3584,0.016784000396728515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,3072,0.016454399625460307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2560,0.016107733050982155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,2048,0.015801599621772765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1536,0.015587199727694193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,1024,0.015332266688346863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,768,0.015095466375350952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,512,0.014916266997655234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,256,0.014713600277900696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,128,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,64,0.014619732896486918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,128,32,0.014694399634997048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,65536,0.019160532951354982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,16384,0.018551466862360637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,10240,0.018631466229756675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,12288,0.018355200688044228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,8192,0.018220800161361694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,7168,0.018837332725524902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,6144,0.018333866198857626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,5120,0.01762346625328064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,4096,0.016852267583211265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3584,0.016594133774439492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2560,0.016115199526151022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,2048,0.015960533420244852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,3072,0.016472533345222473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1536,0.015236266454060874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,1024,0.015078399578730264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,512,0.01497066617012024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,768,0.015099733074506124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,256,0.014910933375358582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,128,0.014467199643452963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,32,0.01458346645037333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,64,64,0.014536533753077188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,65536,0.019501866896947224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,16384,0.018506666024525963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,12288,0.018294399976730345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,10240,0.018652800718943277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,8192,0.018078933159510292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,7168,0.018820265928904213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,6144,0.01837973395983378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,4096,0.016985599199930826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,5120,0.017502933740615845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3584,0.01662506659825643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,3072,0.016463999946912132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2048,0.015954132874806723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,2560,0.0161461333433787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1536,0.015373866756757101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,1024,0.01502293348312378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,512,0.01480959951877594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,768,0.015085867047309876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,256,0.014684800306955972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,128,0.01455893317858378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,32,0.014616533120473226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8,1,32,64,0.014552533626556396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,7168,6.964295450846355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,8192,7.749110412597656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,10240,10.21952412923177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,12288,11.790414428710937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,5120,4.706272888183594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,6144,5.904175821940104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,4096,3.882405344645182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,16384,15.907199096679687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3584,3.434893798828125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2048,2.195484924316406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,2560,2.4500053405761717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1536,1.7437674204508464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,3072,2.9582730611165364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,1024,1.2371412913004556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,256,0.7754165649414062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,512,0.902284812927246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,128,0.7070976257324219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,65536,768,1.11376101175944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,8192,1.9288458506266277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,10240,2.358441670735677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,12288,2.924102274576823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,16384,4.085117848714193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,7168,1.7109099070231122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,6144,1.4148789723714192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,5120,1.1735146840413413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,4096,0.9520266850789388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2560,0.6297738393147786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3072,0.7356704076131184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,3584,0.8686922709147135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,2048,0.5388352076212566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1536,0.43720639546712237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,768,0.30502398808797204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,1024,0.3317461331685384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,512,0.24021333058675132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,128,0.18602773348490398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,256,0.2034506638844808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,10240,1.8421738942464192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,12288,2.295507303873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,16384,2.985401662190755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,7168,1.260248565673828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,8192,1.4286837259928384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,6144,1.0759765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,16384,65536,18.00505574544271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,4096,0.7370751698811848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,5120,0.9018762588500977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3584,0.6489834467569987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,3072,0.5649109522501627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2560,0.48092158635457355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,2048,0.41722879409790037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1536,0.33531627655029295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,512,0.18442667325337728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,768,0.2272874673207601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,256,0.15511679649353027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,1024,0.2539264043172201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,128,0.140229336420695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,10240,1.6129770914713544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,12288,1.8148863474527996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,16384,2.4572265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,12288,65536,13.120160929361978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,7168,1.035866673787435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,8192,1.2121568044026694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,6144,0.9031424204508463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,5120,0.7521439870198567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,4096,0.6154101053873698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3584,0.5584437052408855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,3072,0.4772757212320964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2560,0.4094239870707194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1536,0.28551359176635743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,2048,0.3453280131022135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,768,0.1919167995452881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,1024,0.21483413378397623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,256,0.13042346636454266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,512,0.1553877353668213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,128,0.1187018632888794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,10240,1.213489023844401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,12288,1.4876309712727864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,16384,2.0338282267252605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,10240,65536,10.38049825032552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,7168,0.8711488087972006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,8192,0.9661205291748047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,6144,0.7480469385782877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,5120,0.6282869338989258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3584,0.4525141398111979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,4096,0.5185525258382161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,3072,0.3978773434956869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2560,0.3374549229939779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1536,0.23862826029459633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,2048,0.2889247894287109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,1024,0.1778538703918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,256,0.10703573226928711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,512,0.12910292943318685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,768,0.15924266179402669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,128,0.09800426959991455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,10240,1.0964703877766928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,12288,1.3182933807373047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,16384,1.7937333424886066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,8192,65536,8.334915161132812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,8192,0.8639071782430013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,7168,0.7681642532348633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,6144,0.6775786717732747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,5120,0.5592469533284505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3584,0.4131519953409831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,4096,0.4591466585795085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,3072,0.36375999450683594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2560,0.30456854502360026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1536,0.21236906051635743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,2048,0.259059206644694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,1024,0.16589120229085286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,768,0.14674986203511556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,256,0.09708373546600342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,512,0.1182037353515625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,128,0.08848746617635092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,10240,0.9637226740519205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,12288,1.1330591837565103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,16384,1.567741902669271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,7168,65536,7.45796610514323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,7168,0.6651744206746419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,8192,0.771613883972168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,6144,0.5848778406778972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,5120,0.4877461433410645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,4096,0.39990078608194984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3584,0.36320107777913413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,3072,0.3107658704121908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2560,0.2682506561279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,2048,0.22547094027201334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1536,0.1872149308522542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,768,0.12542186578114828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,1024,0.14062506357828777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,256,0.0839242696762085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,512,0.10131733417510987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,128,0.0772607962290446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,10240,0.828436279296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,12288,0.9653909047444662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,16384,1.3153556823730468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,6144,65536,6.379042053222657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,7168,0.5740544001261394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,8192,0.6560650507609049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,6144,0.5054101308186849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,5120,0.4218720118204753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,4096,0.34496212005615234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3584,0.3131807963053385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2560,0.23263360659281412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,3072,0.27147731781005857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1536,0.16083946228027343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,2048,0.19538240432739257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,768,0.10876693725585937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,1024,0.12443093458811443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,256,0.07229226430257162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,512,0.08878933588663737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,128,0.06474666595458985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,10240,0.689683214823405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,12288,0.8170090357462565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,16384,1.0941205342610678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,5120,65536,5.541234334309896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,7168,0.48122240702311203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,8192,0.5536127726236979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,6144,0.4186581293741862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,5120,0.3517855962117513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,4096,0.2893397331237793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3584,0.2596757411956787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2560,0.1952565352121989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,3072,0.2236469268798828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,2048,0.16300053596496583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1536,0.13282132943471273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,768,0.08932906786600749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,1024,0.10232960383097331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,256,0.05877653360366821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,512,0.07220160166422526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,128,0.05428586800893148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,10240,0.620953623453776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,12288,0.746232541402181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,16384,1.0086869557698568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,4096,65536,4.418413798014322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,7168,0.43710934321085615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,8192,0.4940191904703776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,5120,0.3213120142618815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,6144,0.3845674514770508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,4096,0.26278506914774574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3584,0.23821867307027184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2560,0.17784639994303386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,3072,0.2071839968363444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1536,0.1220362663269043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,2048,0.1476970672607422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,1024,0.0935103972752889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,768,0.08216213385264079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,256,0.05355093479156494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,512,0.06695466836293539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,128,0.04792319933573405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,10240,0.5563968022664387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,12288,0.6496586481730143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,16384,0.863975461324056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3584,65536,4.441351318359375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,8192,0.44379412333170576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,7168,0.3853450775146484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,5120,0.28511571884155273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,6144,0.34020799001057944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,4096,0.2351605256398519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3584,0.2088010629018148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2560,0.15771946907043458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,3072,0.1838197390238444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,2048,0.13243412971496582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1536,0.107315198580424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,1024,0.08343786398569743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,768,0.07298346360524496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,256,0.047278932730356854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,512,0.05826453367869059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,128,0.04236053228378296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,10240,0.48238506317138674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,12288,0.5828128178914388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,16384,0.7579242706298828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,3072,65536,3.5740491231282556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,8192,0.3873642603556315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,7168,0.3440021197001139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,5120,0.2514026641845703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,6144,0.29591147104899085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,4096,0.20542613665262857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3584,0.18521493275960285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2560,0.13901440302530926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,3072,0.16164587338765463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1536,0.09494720300038656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,2048,0.11611200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,1024,0.0732479969660441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,768,0.06375786860783896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,256,0.040863998730977374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,512,0.05071146488189697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,128,0.03659626642862956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,10240,0.41124906539916994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,12288,0.4975882530212402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,16384,0.6577013015747071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2560,65536,3.187508392333984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,8192,0.331388791402181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,7168,0.2952095985412598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,5120,0.2172309398651123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,6144,0.25554240544637047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,4096,0.17684586842854816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3584,0.15900160471598307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2560,0.11852693557739258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,3072,0.13956373532613117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1536,0.0799605369567871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,2048,0.09829333623250326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,1024,0.06174399852752686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,768,0.05426559845606486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,256,0.0339850664138794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,512,0.042431998252868655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,128,0.030473599831263225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,10240,0.3541184107462565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,12288,0.42070614496866865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,16384,0.5447285334269206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,2048,65536,2.636514027913411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,8192,0.28213440577189125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,7168,0.250491730372111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,5120,0.18598613739013672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,6144,0.2163306713104248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,4096,0.15062932968139647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3584,0.13587306340535482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2560,0.1010698636372884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,3072,0.1194218635559082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1536,0.06791679859161377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,2048,0.08361173470815023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,768,0.04508586724599202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,1024,0.05240533351898193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,256,0.027944533030192058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,512,0.03568640152613322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,128,0.024125866095225015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,10240,0.2888021469116211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,12288,0.34276053110758464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,16384,0.44777812957763674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1536,65536,2.2315935770670574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,7168,0.20609599749247232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,8192,0.2331519921620687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,5120,0.15182186762491862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,6144,0.17764800389607746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3584,0.11010453701019288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,4096,0.12353493372599285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2560,0.08071786562601725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,3072,0.0963040033976237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1536,0.054670933882395426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,2048,0.06815146605173747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,768,0.03532053232192993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,1024,0.04184746742248535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,256,0.021052799622217813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,128,0.018074667453765868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,512,0.027267199754714967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,10240,0.2554698626200358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,12288,0.3062570571899414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,1024,65536,1.7518474578857421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,16384,0.4016000111897786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,8192,0.2076021353403727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,7168,0.18466240564982098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,5120,0.13418240547180177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,6144,0.15776106516520183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,4096,0.10900373458862304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3584,0.09818560282389323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2560,0.07143999735514323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,3072,0.0844213326772054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1536,0.048547200361887616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,2048,0.05987733205159505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,1024,0.03642239967981974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,768,0.02993920048077901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,128,0.015141333142916361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,256,0.01764906644821167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,512,0.023451733589172363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,10240,0.22724480628967286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,12288,0.2695136070251465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,768,65536,1.5630507151285806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,16384,0.3514464060465495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,7168,0.16208853721618652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,8192,0.18475947380065919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,5120,0.11663680076599121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,6144,0.1398741404215495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,4096,0.09553813139597575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3584,0.08543252944946289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2560,0.06263786554336548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,3072,0.07361280123392741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1536,0.04195093313852946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,1024,0.030615466833114623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,768,0.024740266799926757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,2048,0.052065066496531164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,128,0.012284800410270691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,256,0.014661332964897156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,512,0.019604265689849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,10240,0.19972373644510905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,12288,0.23794453938802085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,512,65536,1.3671979268391927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,16384,0.31203091939290367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,8192,0.1618293285369873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,7168,0.1418282667795817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,5120,0.10241386890411378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,6144,0.12192959785461426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,4096,0.0833727995554606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2560,0.05388480027516683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3584,0.07371520201365153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,3072,0.06344319979349772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1536,0.0349727988243103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,1024,0.024286933739980063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,2048,0.044326400756835936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,768,0.019980800151824952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,128,0.009257599711418152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,256,0.011452800035476685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,512,0.015844266613324484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,10240,0.1920128027598063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,12288,0.22931946118672691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,256,65536,1.190549341837565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,16384,0.30337387720743814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,8192,0.15516053835550944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,7168,0.13715945879618327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,6144,0.11761173407236736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,5120,0.09858346780141194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,4096,0.07942399978637696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3584,0.0702730655670166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,3072,0.06011626720428467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2560,0.05094826618830363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,768,0.018067200978597008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1024,0.02214933236440023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,2048,0.04123626550038655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,1536,0.032323199510574344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,256,0.010146133104960124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,128,0.008006399869918824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,512,0.014360533157984415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,8192,3.781640625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8192,128,65536,1.1550411224365233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,10240,5.02752431233724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,12288,6.110696411132812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,16384,8.066715494791667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,7168,3.3802197774251304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,6144,2.8006805419921874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,4096,1.9277333577473956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,5120,2.353893280029297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1536,0.827238400777181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2048,1.0478783925374349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3584,1.7005631764729816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,2560,1.1851008097330729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,3072,1.458245340983073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,768,0.5737087885538737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,1024,0.6170368194580078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,256,0.3902741432189941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,128,0.35921707153320315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,65536,512,0.45336958567301433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,8192,0.9104000091552734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,10240,1.178774388631185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,12288,1.346710459391276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,16384,1.798087437947591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,4096,0.48598079681396483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,7168,0.8090304056803385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,6144,0.7033941268920898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,5120,0.5891434351603191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3072,0.37149972915649415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2560,0.31998828252156575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,2048,0.27430826822916665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,3584,0.43144426345825193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,512,0.12433813412984211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1024,0.1743765354156494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,1536,0.2242314656575521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,768,0.15612160364786784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,256,0.10546879768371582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,128,0.0981813351313273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,10240,0.8818410873413086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,12288,1.0550965627034505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,16384,1.413869857788086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,8192,0.6984885533650715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,7168,0.6156362533569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,6144,0.5342271804809571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,5120,0.45331201553344724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,4096,0.3720746676127116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3584,0.33042987187703454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,3072,0.28942505518595374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,16384,65536,8.166829935709636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2048,0.21043732961018882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,2560,0.24694080352783204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1024,0.13522027333577474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,1536,0.17525119781494142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,512,0.09779626528422038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,768,0.11797013282775878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,256,0.08230613072713217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,128,0.07628586292266845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,10240,0.7467519760131835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,12288,0.9086794535319009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,16384,1.19485232035319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,12288,65536,6.525321451822917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,8192,0.592854372660319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,7168,0.5295978546142578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,6144,0.4508885383605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,5120,0.38279787699381507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3584,0.282749875386556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2560,0.21138453483581543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,4096,0.31525119145711267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,3072,0.24354987144470214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,2048,0.18015680313110352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1536,0.14892053604125977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,1024,0.11283520062764485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,768,0.10093546708424886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,512,0.08207573095957438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,256,0.06968959967295328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,128,0.06447466611862182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,10240,0.6086207707722981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,12288,0.7271968205769856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,16384,1.0021013259887694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,8192,0.4835103988647461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,7168,0.4279210726420085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,6144,0.37993812561035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,10240,65536,5.144560241699219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,5120,0.3170741399129232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3584,0.23370772997538247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,3072,0.2021045366923014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,4096,0.2588874657948812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2560,0.1749567985534668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,768,0.08250880241394043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,2048,0.14889492988586425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1536,0.12352320353190105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,1024,0.09353280067443848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,512,0.06790719827016195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,128,0.053653331597646084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,256,0.05717440048853556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,10240,0.5470303853352865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,12288,0.6442058563232422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,16384,0.8498069127400717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,8192,65536,4.033232116699219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,8192,0.4386912027994792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,7168,0.3819210688273112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,6144,0.3322784105936686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,5120,0.2784885406494141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3584,0.20781760215759276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,4096,0.2337674617767334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,3072,0.18062507311503093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2560,0.15679787000020345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1536,0.10971626440684001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,2048,0.13317759831746417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,1024,0.08518826961517334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,768,0.07590186595916748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,512,0.06202453374862671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,256,0.052129065990448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,128,0.04716053406397502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,10240,0.47290347417195633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,12288,0.5585386912027995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,16384,0.7730656305948893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,7168,65536,3.5242070515950523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,5120,0.24550399780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,8192,0.3772490819295248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,7168,0.33264961242675783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,6144,0.2953845342000326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3584,0.18008534113566083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,4096,0.20269333521525063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,3072,0.15671253204345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2560,0.13784640630086262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1024,0.07374613285064698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,2048,0.11526292959849041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,1536,0.0965173323949178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,768,0.06547840038935343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,128,0.04179413318634033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,512,0.05363306601842245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,256,0.04457600116729736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,10240,0.4163413365681966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,12288,0.48265492121378584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,16384,0.640880012512207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,6144,65536,3.0436052958170574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,8192,0.32381439208984375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,7168,0.2868351936340332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,6144,0.2527082602183024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,5120,0.21254612604777018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,4096,0.17503679593404134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3584,0.15735573768615724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,3072,0.136845858891805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2560,0.11995200316111247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1024,0.065011199315389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,1536,0.08286720116933187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,2048,0.09989226659138997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,512,0.04703573385874431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,768,0.05783040126164755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,256,0.03921706676483154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,128,0.03572373390197754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,10240,0.34158719380696617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,16384,0.5389226913452149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,12288,0.40224533081054686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,5120,65536,2.691086832682292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,6144,0.20950080553690592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,8192,0.2710229237874349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,7168,0.23984427452087403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,5120,0.17983253796895343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3072,0.11383999983469646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,3584,0.13010453383127848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,4096,0.14584320386250813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2560,0.10059306621551514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,2048,0.08401280244191488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1536,0.06946559747060141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,1024,0.05352426767349243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,768,0.047254399458567305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,512,0.03895999987920125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,128,0.02981013258298238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,256,0.03201919992764791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,10240,0.3148437182108561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,12288,0.3665898640950521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,16384,0.4799327850341797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,4096,65536,2.2122528076171877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,8192,0.248091729482015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,7168,0.21991252899169922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,6144,0.1924575964609782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,5120,0.16358720461527507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3584,0.12081066767374675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,3072,0.10447680155436198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,4096,0.13461546897888182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2560,0.09215466976165772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1536,0.06378986835479736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,1024,0.04938773314158122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,2048,0.07629760106404623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,512,0.03591786623001099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,768,0.043806934356689455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,256,0.029445334275563555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,128,0.026741333802541095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,10240,0.2738666534423828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,16384,0.43186025619506835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,12288,0.32436908086140953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3584,65536,2.059701283772786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,8192,0.2206933339436849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,7168,0.19696106910705566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,6144,0.17113812764485675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,5120,0.14678826332092285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3072,0.09236799875895182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,3584,0.1078389326731364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,4096,0.11959573427836101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2560,0.08111146291097006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,2048,0.06837226549784342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1536,0.05719893376032511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,1024,0.044278399149576826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,768,0.03873279889424642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,512,0.031787733236948654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,256,0.026231465737024943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,128,0.023596799373626708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,10240,0.2441962718963623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,12288,0.28908586502075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,16384,0.37532265981038415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,3072,65536,1.7350165049235027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,8192,0.1950826644897461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,7168,0.1739850680033366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,6144,0.1513312021891276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,5120,0.12821439901987713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2560,0.07134719689687094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,4096,0.10443946520487468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3584,0.09420159657796225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,3072,0.08150293032328287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,2048,0.06125119924545288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1024,0.039185067017873124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,768,0.03375360171000163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,1536,0.0503658652305603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,256,0.022241065899531044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,512,0.02765013376871745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,128,0.020438400904337566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,10240,0.210044797261556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,12288,0.24837333361307778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,16384,0.32995840708414714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2560,65536,1.5794420878092448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,6144,0.13128639856974283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,8192,0.16926080385843914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,7168,0.15028586387634277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,5120,0.11256852944691975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,4096,0.09037333329518636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3072,0.07015466690063477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,3584,0.08053119977315268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2560,0.06149653196334839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,2048,0.052515200773874914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1024,0.03285546700159709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,1536,0.0439082662264506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,768,0.02890239953994751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,512,0.02316373387972514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,128,0.0174453337987264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,256,0.018734933932622273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,10240,0.18327466646830243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,12288,0.21237333615620932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,2048,65536,1.2969322204589844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,16384,0.2770282745361328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,8192,0.14601492881774902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,7168,0.12828053633371989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,6144,0.11149653593699138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,5120,0.09496959845225016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3072,0.0604042649269104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,4096,0.07762880325317383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2560,0.05272639989852905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,3584,0.06864319642384847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,2048,0.0448309342066447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1024,0.027808000644048054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,768,0.023890133698781332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,1536,0.03640640179316203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,512,0.019938133160273232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,256,0.015913599729537965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,128,0.014632532993952433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,10240,0.14704000155131022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,12288,0.17410240173339844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1536,65536,1.0957450866699219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,16384,0.22751466433207193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,7168,0.10481493473052979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,8192,0.11894400119781494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,5120,0.0760853370030721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,6144,0.09139413038889567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3072,0.049073068300882976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,3584,0.056170666217803956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2560,0.04332586526870728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,4096,0.06266560157140097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,2048,0.036266668637593584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,768,0.01938026746114095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1536,0.029410133759180706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,1024,0.02220266660054525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,512,0.01588586668173472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,128,0.011709866921106975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,256,0.012819199760754903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,1024,65536,0.8765888214111328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,10240,0.13891414006551106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,12288,0.16484053929646808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,16384,0.217140261332194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,6144,0.085098663965861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,8192,0.11204053560892742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,7168,0.09852159818013509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,5120,0.07192959785461425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,4096,0.05909119844436646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3072,0.045561599731445315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,3584,0.0525546669960022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2560,0.040830934047698976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1536,0.02590399980545044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,768,0.016773333152135216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,1024,0.020360533396402994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,2048,0.03325973351796468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,512,0.013912533720334372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,128,0.009684266646703084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,256,0.010814932982126872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,768,65536,0.8455146789550781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,8192,0.09267199834187825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,10240,0.11434666315714519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,12288,0.13618879318237304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,16384,0.18024640083312987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,7168,0.08186559677124024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,6144,0.07071466445922851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,5120,0.05997546513875326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,4096,0.04946240186691284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3584,0.04441599845886231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2560,0.03362346490224202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,3072,0.03871786594390869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,2048,0.02790293296178182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1536,0.02200746734937032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,1024,0.016821332772572837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,512,0.012172800302505494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,768,0.01455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,256,0.009554133812586466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,128,0.00865280032157898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,512,65536,0.6872917175292969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,10240,0.10537280241648357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,12288,0.12455253601074219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,16384,0.16404587427775066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,8192,0.08420906861623129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,7168,0.07523946762084961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,5120,0.05375466744105021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,6144,0.06391573349634806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3072,0.03431146542231242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2560,0.02986133297284444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,4096,0.044215468565622966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,3584,0.04012480179468791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,768,0.01260373294353485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1024,0.014962133765220643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,2048,0.024220800399780272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,1536,0.019556266069412232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,256,0.008507733543713886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,512,0.010517332951227825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,128,0.007624533275763195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,256,65536,0.6249834696451824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,10240,0.10256106853485107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,12288,0.12371306419372559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,8192,0.08308373292287191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,16384,0.16007787386576336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,7168,0.07245333194732666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,6144,0.06345599889755249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,5120,0.052566401163736974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,4096,0.04262293179829915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2048,0.022696532805760703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3072,0.03294080098470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,3584,0.038254932562510176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,2560,0.02775040070215861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,768,0.011707733074824016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1024,0.013923199971516928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,512,0.009711999694506328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,1536,0.018447999159495035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,256,0.0075647999842961625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,128,0.0067221333583196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4096,128,65536,0.6103306452433268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,8192,1.8147275288899738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,10240,2.2419466654459637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,7168,1.570899200439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,12288,2.8653513590494795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,16384,4.130094909667969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3584,0.8154474894205729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,4096,0.9335807800292969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,6144,1.3656906127929687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,5120,1.1337141672770181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2048,0.5240906715393067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,2560,0.5951424280802409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1024,0.3149792035420736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,1536,0.42026561101277665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,3072,0.70044371287028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,256,0.19984639485677083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,128,0.18387093544006347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,768,0.29092159271240237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,65536,512,0.232916259765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,8192,0.45481494267781575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,12288,0.6766880035400391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,10240,0.5778570810953776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,16384,0.8999872207641602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,6144,0.35153172810872396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,4096,0.24169707298278809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,5120,0.2978101412455241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,7168,0.4020949363708496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2048,0.13854613304138183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3072,0.19026560783386232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,2560,0.1656768004099528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,3584,0.22039039929707846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,768,0.08127146561940511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1536,0.11512959798177083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,512,0.06594133377075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,1024,0.0904970645904541
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,256,0.056354133288065586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,128,0.0526581327120463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,12288,0.5275050799051921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,16384,0.6913290659586588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,10240,0.433135986328125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,6144,0.266924794514974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,7168,0.30801280339558923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,8192,0.3579530715942383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,4096,0.18590505917867023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,16384,65536,4.146711476643881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,5120,0.22537172635396324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3584,0.16933333079020182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2048,0.1066314697265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,2560,0.12770666281382242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,3072,0.1441706657409668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1536,0.08936639626820883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,1024,0.0693290630976359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,512,0.05161813497543335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,256,0.04375466505686442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,768,0.06264319817225138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,128,0.040931200981140135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,10240,0.36564372380574545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,12288,0.4426591873168945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,16384,0.6019093195597331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,12288,65536,3.160959879557292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,8192,0.2940608024597168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,7168,0.26323092778523766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,6144,0.2316746711730957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,5120,0.19338879585266114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,4096,0.15730346043904622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3072,0.12469333012898762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,3584,0.14349546432495117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2048,0.09102613131205241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,2560,0.11075092951456707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1536,0.07635733286539713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,768,0.05316053231557211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,1024,0.060090665022532144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,512,0.044595201810201004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,256,0.03771200180053711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,128,0.03521813154220581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,12288,0.36596266428629554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,10240,0.2985151926676432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,10240,65536,2.427416483561198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,16384,0.4816202799479167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,8192,0.24810880025227866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,5120,0.16018026669820148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,7168,0.21337386767069497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,6144,0.18816213607788085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3072,0.10146880149841309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,4096,0.13235092957814534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,3584,0.11853546301523846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1536,0.06332799990971884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2560,0.09018239974975586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,2048,0.075382399559021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,1024,0.05035306612650553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,768,0.044139734903971356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,512,0.037393065293629964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,256,0.03165653347969055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,128,0.029448533058166505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,12288,0.3169077237447103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,10240,0.27421865463256834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,16384,0.42124799092610676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,8192,65536,1.9592405954996746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,8192,0.21468159357706704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,7168,0.1928256034851074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,6144,0.16829652786254884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,5120,0.14231252670288086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,4096,0.1163413365681966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3584,0.106440536181132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,3072,0.09303999741872152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2560,0.080621870358785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1536,0.05767253239949545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,2048,0.06798293590545654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,1024,0.045906134446461994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,768,0.040676267941792806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,512,0.03382720152537028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,256,0.028870399792989093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,128,0.026667733987172444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,12288,0.27676480611165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,7168,65536,1.7257439931233722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,16384,0.37349440256754557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,10240,0.23605333964029945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,6144,0.14832852681477865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,8192,0.1881290594736735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,7168,0.17176639238993327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,5120,0.12485439777374267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,4096,0.10447999636332195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3584,0.09194666544596354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,3072,0.08142613569895427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2560,0.07034239768981934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,2048,0.060229333241780605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1536,0.05017600059509277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,1024,0.039656531810760495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,768,0.03477120002110799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,512,0.029465599854787188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,256,0.024918399254480996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,128,0.02279040018717448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,6144,65536,1.4902837117513021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,12288,0.23944320678710937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,10240,0.20784853299458822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,16384,0.316102409362793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,8192,0.16575679779052735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,6144,0.12906560103098552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,7168,0.1446986675262451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,5120,0.10929493109385173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3584,0.08097600142161052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,4096,0.09003732999165853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,3072,0.07016426722208659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2560,0.06130346854527792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1536,0.04475413163503011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,512,0.026232532660166424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,2048,0.0528277317682902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,1024,0.035113600889841716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,768,0.030936533212661745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,128,0.019801600774129232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,256,0.02203306754430135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,5120,65536,1.2437408447265625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,12288,0.20707093874613441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,10240,0.17318933804829914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,16384,0.26808748245239256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,5120,0.09026880264282226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,7168,0.12488213380177815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,8192,0.1372767925262451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,6144,0.10826026598612468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3584,0.06738666693369547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,4096,0.07508479754130046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2560,0.05157866477966309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,3072,0.05875840187072754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1536,0.0370410680770874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,2048,0.04461439847946167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,256,0.018268799781799315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,768,0.025568000475565594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,1024,0.029449599981307983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,512,0.021262933810551964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,128,0.016663466890652977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,4096,65536,1.0486613591512044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,12288,0.2058560053507487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,16384,0.27771199544270836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,10240,0.17670079867045085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,8192,0.1401685396830241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,7168,0.1236074686050415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,6144,0.10696533521016438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,5120,0.09113600254058837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,4096,0.07444266478220621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3072,0.057231998443603514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,3584,0.06468053261439005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2560,0.05040640036265055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,2048,0.04176533222198486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,65536,1.0799551645914713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1536,0.03323946595191955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,1024,0.026655999819437663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,768,0.021397332350413002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,256,0.014828800161679586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,512,0.017359999815622966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3584,128,0.012632532914479574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,8192,0.11419306596120198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,10240,0.14061759312947592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,16384,0.21883412996927895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,12288,0.16679360071818033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,7168,0.10148906707763672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,4096,0.06163626511891683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,5120,0.07449386914571127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,6144,0.08753920396169027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3584,0.05534079869588217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2560,0.04252693255742391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1536,0.030850134293238324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,3072,0.04868373473485311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,2048,0.036824532349904376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,256,0.015517866611480713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,1024,0.024269866943359374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,768,0.021333332856496176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,512,0.018067200978597008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,128,0.014006400108337402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,3072,65536,0.8473215738932292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,12288,0.18315626780192057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,10240,0.147486941019694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,16384,0.22186880111694335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,8192,0.11702933311462402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,7168,0.10392213662465413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,6144,0.08730026880900064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,5120,0.0756608009338379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,4096,0.06028480132420858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3072,0.04644266764322917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,3584,0.0556330680847168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2560,0.041630931695302326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,65536,0.8952149073282877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1536,0.026792534192403156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,2048,0.034409598509470625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,1024,0.02179093360900879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,768,0.017191465695699057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,512,0.01402453382809957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,256,0.01188373366991679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2560,128,0.009965866804122925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,12288,0.12943680286407472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,8192,0.08635413646697998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,16384,0.1684991995493571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,10240,0.10744319756825764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,7168,0.07726826667785644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,4096,0.04779200156529744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,5120,0.05656426747639974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,6144,0.06704746882120768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3584,0.042873601118723556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,3072,0.0380181352297465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2560,0.032569599151611325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1536,0.0235317329565684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,2048,0.028152533372243244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,512,0.013980799913406372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,1024,0.018554667631785073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,256,0.012018133203188579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,768,0.016446933150291443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,128,0.010856533050537109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,2048,65536,0.6381653467814128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,12288,0.11743786334991455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,10240,0.10003306865692138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,16384,0.15316373507181805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,8192,0.08082026640574137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,7168,0.07029653390248616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,6144,0.06016000111897787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,5120,0.052741332848866784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,4096,0.04319786628087362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3584,0.03918079932530721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,3072,0.033505066235860186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2560,0.029659734169642134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,65536,0.5947861353556315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,2048,0.02534079949061076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1536,0.02051946719487508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,1024,0.01627626617749532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,768,0.013739732901255288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,512,0.011905066172281901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,256,0.01018773317337036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1536,128,0.008884267012278239
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,12288,0.08871040344238282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,10240,0.07425599892934164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,16384,0.11565972963968914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,8192,0.06075199842453003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,4096,0.03357760111490886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,5120,0.040999468167622885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,6144,0.04692693154017131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,7168,0.053939199447631835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3072,0.026649600267410277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,3584,0.030060799916585286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2560,0.02304853399594625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1024,0.01365546683470408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,2048,0.01969919999440511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,1536,0.01649386684099833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,128,0.007971199850241344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,512,0.010442666212717692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,256,0.009097599983215332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,768,0.01181653340657552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,1024,65536,0.4457738558451335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,12288,0.08359359900156657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,10240,0.07074346542358398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,16384,0.11243306795756022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,8192,0.057942398389180506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,7168,0.05112959941228231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,6144,0.044327465693155925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,5120,0.03915946483612061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,4096,0.03132266600926717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3584,0.028172800938288372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,3072,0.024730666478474935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2560,0.021939200162887574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,2048,0.018296533823013307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1536,0.015188266833623251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,1024,0.012308266758918763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,768,0.010776533683141073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,512,0.009245866537094116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,256,0.008099199831485748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,128,0.007065600156784058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,768,65536,0.4220287958780925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,12288,0.07750293413798014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,8192,0.052765866120656334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,10240,0.06591253280639649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,16384,0.10124266942342122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,6144,0.0413696010907491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,4096,0.028753066062927247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,5120,0.03556160132090251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,7168,0.047014399369557695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2048,0.016812799374262492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3072,0.022366933027903237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,3584,0.025859200954437257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,2560,0.0200053334236145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,65536,0.3796789487202962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1024,0.01146666705608368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,512,0.008694400389989216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,768,0.009886933366457622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,256,0.007885866860548655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,1536,0.014354133605957031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,512,128,0.006867200136184692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,10240,0.06040106614430746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,12288,0.07105813026428223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,16384,0.09306666851043702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,8192,0.049092264970143636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,6144,0.03790719906489055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,7168,0.043331201871236166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,5120,0.03221759994824727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,4096,0.025846399863560993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3072,0.020492800076802573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,3584,0.02355519930521647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2560,0.018476800123850504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,2048,0.015511467059453329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,65536,0.35141121546427406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1024,0.010220799843470256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,1536,0.013025066256523133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,512,0.007925333579381307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,768,0.009013332923253377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,256,0.007085866729418437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,256,128,0.00613973339398702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,12288,0.06895999908447266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,16384,0.09001173178354899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,10240,0.05786986748377482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,8192,0.04727360010147095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,7168,0.0421834667523702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,4096,0.025036799907684325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,6144,0.03676799933115642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,5120,0.030687999725341798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3584,0.022873600323994957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,3072,0.01988906661669413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2048,0.014869333306948344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,2560,0.017733333508173625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,65536,0.33986132939656577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1024,0.009945600231488546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,1536,0.012802132964134216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,512,0.0076682666937510175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,768,0.008706133564313252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,256,0.006884266436100006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2048,128,128,0.0058890665570894875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,7168,0.791425069173177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,8192,0.8848031997680664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,10240,1.1017173767089843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,12288,1.4248128255208334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,4096,0.46292587916056316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,5120,0.5597824096679688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3584,0.4218549410502116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,16384,1.8891284942626954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,6144,0.6753642400105794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1536,0.21846720377604165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2048,0.26957438786824545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,2560,0.306989860534668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,1024,0.16326826413472492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,3072,0.3595744132995605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,256,0.10456746419270832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,128,0.0964202642440796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,512,0.12237652937571208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,65536,768,0.15115413665771485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,10240,0.29017814000447595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,8192,0.23030932744344076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,12288,0.34037440617879233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,16384,0.4421034812927246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,7168,0.2092192014058431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,6144,0.1773962656656901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,5120,0.1486933390299479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,4096,0.12245972951253255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3584,0.11206826368967693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2048,0.07235413392384847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,3072,0.09821013609568277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,2560,0.08448639710744223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1536,0.06067519982655843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,256,0.03134933312733968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,1024,0.04848106702168782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,768,0.04408106803894043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,512,0.03631786505381267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,128,0.029105067253112793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,10240,0.22281600634256998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,12288,0.26014720598856605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,16384,0.3451242764790853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,8192,0.18122560183207195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,7168,0.1574399948120117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,16384,65536,2.0345994313557942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,6144,0.13689707120259603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,5120,0.11647253036499024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3072,0.07481280167897543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,3584,0.08578773339589438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,4096,0.09494826793670655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2560,0.06493653456370035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1536,0.048086400826772055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,2048,0.05647679964701334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,512,0.02894933422406514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,1024,0.03760639826456706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,768,0.033758934338887533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,256,0.024946133295694985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,128,0.023280000686645506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,12288,65536,1.354317855834961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,16384,0.2993109385172526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,10240,0.18570133845011394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,12288,0.21871573130289712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,7168,0.1356895923614502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,8192,0.1510111967722575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,6144,0.11787839730580647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,5120,0.09748266537984213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3072,0.06391573349634806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2560,0.05545599857966105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,3584,0.07192746798197427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,4096,0.08231786886850992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1536,0.04100906848907471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,2048,0.048749868075052896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,1024,0.032409600416819256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,768,0.02935253381729126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,512,0.0250271995862325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,128,0.019581866264343262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,256,0.021640533208847047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,16384,0.23828479448954262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,10240,65536,1.1569727579752604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,12288,0.17985706329345702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,10240,0.15547626813252766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,7168,0.1108394702275594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,8192,0.12317972977956135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,6144,0.09517760276794433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,5120,0.08055893580118814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3584,0.06022826830546061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,3072,0.05333120028177897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,4096,0.06670186519622803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2560,0.04618240197499593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1024,0.027453867594401042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,1536,0.034065067768096924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,2048,0.040746665000915526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,768,0.02503146727879842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,256,0.017760000626246133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,128,0.016537599762280784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,512,0.021268266439437866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,8192,65536,0.9538389205932617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,12288,0.1828106721242269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,10240,0.1558890660603841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,16384,0.24201706250508628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,8192,0.1268725315729777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,6144,0.09490453402201335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,7168,0.11029333273569744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,5120,0.07886826992034912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,4096,0.06742719809214273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3072,0.050810666879018154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,3584,0.058177065849304196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2560,0.04514026641845703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,65536,0.9781002680460611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,2048,0.038031999270121256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1536,0.0301749328772227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,1024,0.02526613275210063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,768,0.02064853310585022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,512,0.01759679913520813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,256,0.014086400469144186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,7168,128,0.012332800030708312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,12288,0.14403626124064128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,10240,0.12278933525085449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,8192,0.09644160270690919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,16384,0.18511145909627277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,7168,0.08646186987559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,6144,0.07598400115966797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,4096,0.05338026682535807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,5120,0.06349440018335978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3072,0.04278826713562012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2048,0.032730666796366374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,3584,0.048198401927948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1536,0.02760853370030721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,2560,0.036984535058339436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,1024,0.022235733270645142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,768,0.020423465967178346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,256,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,512,0.01720959941546122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,128,0.013608533143997192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,6144,65536,0.7240501403808594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,12288,0.14679253896077474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,16384,0.1859658718109131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,10240,0.1339914639790853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,8192,0.10179093678792317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,6144,0.0740991989771525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,7168,0.08823359807332357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,5120,0.0648629347483317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,4096,0.05430506865183512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,65536,0.7558591842651368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3072,0.04043519894282023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,3584,0.04781119823455811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2560,0.03598293463389079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,2048,0.03049066662788391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1536,0.023923200368881226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,768,0.01660693287849426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,512,0.013473066687583923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,1024,0.02038080096244812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,256,0.011061333616574605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,5120,128,0.010021332899729412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,12288,0.10534613132476807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,10240,0.08669226964314779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,16384,0.13546773592631023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,8192,0.07060266335805257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,6144,0.055011200904846194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,7168,0.06322240034739177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,5120,0.04752320051193237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,4096,0.039928531646728514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2560,0.028302933772404986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3584,0.035894401868184406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,3072,0.032315733035405476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1536,0.020935465892155967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,2048,0.02469653288523356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,1024,0.017173333962758382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,768,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,512,0.013490133484204612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,256,0.011482666929562886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,128,0.010780800382296245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,4096,65536,0.5121823946634929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,8192,0.07119359970092773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,16384,0.13823572794596356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,10240,0.089629864692688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,12288,0.1059007962544759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,7168,0.0634112000465393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,6144,0.05464533170064291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,5120,0.04667733510335286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,4096,0.03962986469268799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3584,0.03456639846165975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,3072,0.030663466453552245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2560,0.027420800924301148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,2048,0.023126399517059325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1024,0.01585919956366221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,1536,0.01906986633936564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,65536,0.5509749094645182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,768,0.013364266355832419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,512,0.011482666929562886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,256,0.00955839951833089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3584,128,0.00870293378829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,10240,0.07876479625701904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,12288,0.09324800173441569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,8192,0.065120001633962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,16384,0.1226090669631958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,6144,0.048793598016103105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,7168,0.05641813278198242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,4096,0.035394132137298584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,5120,0.04322666724522908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,65536,0.4806336085001628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3072,0.02786239981651306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,3584,0.03242879907290141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2560,0.025099732478459674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,2048,0.0217685341835022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1536,0.017938133080800375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,512,0.011374933520952861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,1024,0.014541866381963095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,768,0.01290773351987203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,256,0.009577600161234538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,3072,128,0.008668800195058186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,12288,0.08722879886627197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,10240,0.07248853047688802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,16384,0.11279253164927165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,8192,0.060670932133992515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,6144,0.04520959854125976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,7168,0.05223466555277506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,4096,0.032849067449569704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,5120,0.040109864870707196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,65536,0.43756694793701173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3072,0.025912533203760784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,3584,0.02993280092875163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2560,0.023215999205907188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,2048,0.019882667064666747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1536,0.016219733158747356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,768,0.011763200163841248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,512,0.010316800077756245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,1024,0.013425067067146301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,256,0.008687999844551087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2560,128,0.007851733267307282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,10240,0.05599573453267416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,12288,0.06459199984868368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,16384,0.08643093109130859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,8192,0.045108266671498615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,7168,0.041117866834004715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,5120,0.030628265937169392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,4096,0.025663999716440837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,6144,0.03572906653086345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3584,0.024013866980870567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2560,0.01876586675643921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1024,0.011548800269762675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,3072,0.02108906706174215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,2048,0.016526933511098227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,1536,0.013678933183352152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,768,0.010840533177057902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,512,0.009734400113423665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,65536,0.31831893920898435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,256,0.008357333143552144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,2048,128,0.007777066528797149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,16384,0.07947093645731608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,12288,0.05881813367207846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,10240,0.05098346471786499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,8192,0.04251413345336914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,6144,0.03206506570180257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,7168,0.037307735284169516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,4096,0.02377386689186096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,5120,0.028293333450953168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2560,0.01707306702931722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3072,0.019062399864196777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,2048,0.014658133188883463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,3584,0.021721599499384563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1024,0.010279466708501179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,65536,0.30052372614542644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,512,0.00862613320350647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,1536,0.012322133779525757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,768,0.009495466947555542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,128,0.00674773355325063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1536,256,0.007359999914964039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,12288,0.054109867413838705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,16384,0.07036266326904297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,10240,0.047075200080871585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,8192,0.037808001041412354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,4096,0.02170026699701945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,5120,0.02553066611289978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,6144,0.02977386713027954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,7168,0.03331519961357117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,65536,0.26137173970540367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3072,0.017084799210230508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2048,0.013095466295878091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,3584,0.019421867529551187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,2560,0.015440000096956888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1536,0.011485866705576579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,256,0.0071274667978286745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,512,0.008144000172615051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,1024,0.009447466333707173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,768,0.00879039963086446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,1024,128,0.006671999891599019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,12288,0.04937386512756348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,10240,0.04322453339894612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,16384,0.06569493214289347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,8192,0.034619732697804766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,6144,0.02709546685218811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,7168,0.031035733222961426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,5120,0.023546665906906128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,4096,0.01962453325589498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,65536,0.24975573221842445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3072,0.015735466281572977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,3584,0.017602133750915527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2560,0.014196266730626425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,2048,0.012078932921091716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1536,0.010339200496673584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,1024,0.008729599912961324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,512,0.00734506646792094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,768,0.008039466540018718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,256,0.006331733365853627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,768,128,0.005883733431498209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,10240,0.041316266854604086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,12288,0.04788586695988973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,16384,0.06182399988174438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,8192,0.03332266608874003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,7168,0.02881173292795817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,5120,0.02197973330815633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,6144,0.02573866645495097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,4096,0.018627200524012247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,65536,0.2289642651875814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3072,0.015201066931088766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2048,0.011751466989517212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,3584,0.016992000738779704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,2560,0.013548800349235534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1536,0.010101333260536194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,1024,0.008289066453774769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,512,0.007223466535409291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,768,0.007930666704972585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,256,0.0063498665889104204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,512,128,0.00580266664425532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,10240,0.037862400213877365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,12288,0.04542613426844279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,8192,0.030449066559473676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,16384,0.05770026842753092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,5120,0.020707199970881142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,6144,0.02444266676902771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,4096,0.017568000157674155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,7168,0.02720213333765666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,65536,0.21774826049804688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3072,0.01455893317858378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2048,0.011101866761843365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,3584,0.016359466314315795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,2560,0.013078400492668152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1536,0.009727999567985535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,512,0.0070261334379514056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,1024,0.00812906672557195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,768,0.00757120003302892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,256,0.006087466577688853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,256,128,0.005635199944178263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,10240,0.03755946556727092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,8192,0.03025706609090169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,12288,0.04434666633605957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,16384,0.05760000149408976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,6144,0.02379306753476461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,5120,0.020652800798416138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,7168,0.02727893392244975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,4096,0.017517866690953572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3584,0.016238933801651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,3072,0.014262400070826211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2560,0.01290880044301351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,65536,0.21472533543904623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,2048,0.010835199554761251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1536,0.009761066238085429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,1024,0.008055466910203297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,768,0.007649066547552745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,512,0.007065600156784058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,256,0.005995733539263407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1024,128,128,0.005654400090376536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,7168,0.7337696075439453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,8192,0.8417888005574545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,10240,1.0702388763427735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,12288,1.2936490376790366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,16384,1.676638921101888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,6144,0.6337237040201823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,5120,0.5348106384277344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,4096,0.41944745381673176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3584,0.36720641454060876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1536,0.16556053161621093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2048,0.20736533800760903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,2560,0.27311038970947266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,3072,0.3160671869913737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,1024,0.12451199690500896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,256,0.07787626584370931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,128,0.07587093512217204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,512,0.09231039683024088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,65536,768,0.11525333722432454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,10240,0.27844053904215493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,8192,0.20500799814860024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,12288,0.3201471964518229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,16384,0.4130282719930013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,7168,0.18174187342325848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,5120,0.13174506823221843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,4096,0.10612160364786785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,6144,0.16011306444803874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2560,0.07211306889851889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,2048,0.056764801343282066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3584,0.09320426781972249
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,3072,0.0806943972905477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,512,0.02829119960467021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1536,0.04683733383814494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,768,0.03402986526489258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,1024,0.03755840063095093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,65536,1.7335647583007812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,128,0.021874133745829263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,16384,256,0.024321067333221435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,12288,0.23021653493245445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,16384,0.2998954772949219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,10240,0.1950271924336751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,8192,0.15905386606852215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,7168,0.13430933952331542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,6144,0.11564160188039144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,4096,0.07997546990712484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,5120,0.1028714656829834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3072,0.061469864845275876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,3584,0.07261760234832763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2560,0.05393600066502889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,2048,0.04612693389256795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,65536,1.3397141774495442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1536,0.03489493529001872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,512,0.019063466787338258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,768,0.023346134026845298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,1024,0.03054186701774597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,128,0.014005333185195923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,12288,256,0.016603733102480568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,12288,0.19169173240661622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,16384,0.25510506629943847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,10240,0.1796298662821452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,8192,0.13706132570902507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,7168,0.12334187030792236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,6144,0.10138773123423259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,4096,0.07192853291829428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,5120,0.08846293290456136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3072,0.053041064739227296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2560,0.04763840039571126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,3584,0.0630016009012858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,65536,1.0504501342773438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,2048,0.040559999148050946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1536,0.03169706662495931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,256,0.015239466230074564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,768,0.02203413248062134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,512,0.01840106646219889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,1024,0.027085866530736285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,10240,128,0.013448533415794373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,10240,0.14707733790079752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,12288,0.16820586522420247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,16384,0.21033066113789878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,8192,0.10790613492329915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,7168,0.09571839968363444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,6144,0.08287040392557779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,5120,0.06898773511250814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,4096,0.05616533358891805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2560,0.038939734299977616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3584,0.050329601764678954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,3072,0.04395733277002971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,2048,0.031923200686772665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,65536,0.879694938659668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1536,0.026601600646972656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,768,0.019383466243743895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,1024,0.021474132935206093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,256,0.01384213368097941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,512,0.016376533110936484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,8192,128,0.013313066959381104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,12288,0.14106772740681967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,10240,0.13011519908905028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,16384,0.18370134035746258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,8192,0.10008533000946045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,6144,0.07170773347218831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,7168,0.08494186401367188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,4096,0.05031466484069824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,5120,0.06368000109990438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3072,0.038667734464009604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,65536,0.7627583821614583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,3584,0.046003198623657225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2560,0.035097599029541016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,2048,0.03021120031674703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1536,0.023205333948135377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,768,0.01615466674168905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,512,0.01336853305498759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,1024,0.020113066832224528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,256,0.010826667149861652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,7168,128,0.00979306697845459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,12288,0.1478271961212158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,10240,0.12775359948476156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,16384,0.18514240582784017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,8192,0.08665493329366049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,6144,0.07490773200988769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,7168,0.07624853452046712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,4096,0.04470400015513103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,5120,0.057011198997497556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3072,0.035916801293691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,3584,0.04019840161005656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,65536,0.7649589538574219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2560,0.03209173281987508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,2048,0.026692267258961993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1536,0.02158613403638204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,256,0.01092906693617503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,512,0.012642133235931396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,768,0.015705600380897522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,1024,0.018566399812698364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,6144,128,0.009758933385213216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,12288,0.10727573235829671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,10240,0.10178559621175129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,16384,0.1391413370768229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,8192,0.07699840068817139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,6144,0.0554314653078715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,7168,0.0689141352971395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,5120,0.04850560029347738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,4096,0.04117439985275269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,65536,0.5426570892333984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3072,0.030703999598821002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,3584,0.035916801293691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2560,0.02788693308830261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,2048,0.02364586591720581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1536,0.019412267208099365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,1024,0.016416000326474507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,512,0.011416533589363098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,768,0.01386240025361379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,256,0.009776000181833904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,5120,128,0.00906880001227061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,12288,0.10789120197296143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,10240,0.08471253712972006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,16384,0.13502826690673828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,8192,0.060974931716918944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,6144,0.05256746610005697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,7168,0.05853973229726156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,4096,0.0347818652788798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,5120,0.042713598410288496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,65536,0.5515754699707032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3072,0.02749653259913127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,3584,0.030745599667231244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2560,0.024821333090464272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,2048,0.02015786568323771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1536,0.01689280072848002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,1024,0.014443733294804893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,768,0.011768533786137899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,512,0.01023466686407725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,256,0.009016533692677815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,4096,128,0.008202666540940602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,12288,0.09865813255310059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,10240,0.08493226369222005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,16384,0.12931839625040692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,8192,0.059816535313924155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,6144,0.04528426726659139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,7168,0.052593068281809485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,4096,0.03277013301849365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,5120,0.04010879993438721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,65536,0.5278709411621094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3072,0.025298132499059038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,3584,0.029412267605463664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2560,0.023112533489863078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,2048,0.019879466295242308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1536,0.01665173371632894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,1024,0.013520000378290811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,768,0.011852799852689107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,512,0.010104533036549885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,256,0.008648533622423809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3584,128,0.007957333326339721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,10240,0.07893973191579183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,12288,0.09408426284790039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,16384,0.12247573534647624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,8192,0.0572106679280599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,6144,0.04545386632283528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,7168,0.05244693358739218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,5120,0.03778773148854574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,4096,0.0326474666595459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,65536,0.500164254506429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3072,0.02402879993120829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,3584,0.02683093349138896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2560,0.021840000152587892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,2048,0.01830613414446513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1536,0.015032533804575601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,768,0.010874666770299276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,1024,0.012505599856376648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,512,0.00943893293539683
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,256,0.008215466638406117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,3072,128,0.00757120003302892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,12288,0.07420266469319661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,10240,0.06255999803543091
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,16384,0.09916693369547526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,8192,0.04788586695988973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,7168,0.04518826802571614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,6144,0.03970133463541667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,4096,0.028170667092005414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,5120,0.03155840039253235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,65536,0.32147626876831054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3072,0.021660800774892172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,3584,0.023745065927505492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1536,0.013196800152460733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2560,0.019233065843582153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,2048,0.015646933515866598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,1024,0.010838400324185688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,256,0.007299200197060903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,768,0.009533866246541341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,512,0.008144000172615051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2560,128,0.006718933085600535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,12288,0.06742400328318278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,10240,0.05713599920272827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,8192,0.046393601099650066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,16384,0.09173866907755533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,7168,0.04073706865310669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,6144,0.03607786496480306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,4096,0.02517333428064982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,5120,0.030366933345794676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,65536,0.3550357182820638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3072,0.019170133272806804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,3584,0.021974400679270426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2560,0.017192532618840538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,2048,0.014723199605941772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1536,0.011634133259455363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,512,0.00763733337322871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,128,0.006253866851329804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,768,0.008648533622423809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,1024,0.009800533453623455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,2048,256,0.006817066669464111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,12288,0.05925973256429037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,10240,0.049737600485483806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,8192,0.03899946610132853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,16384,0.07969706853230794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,7168,0.03582826852798462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,6144,0.03145813345909119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,4096,0.021335466702779134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,5120,0.024977066119511924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,65536,0.3152885437011719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3072,0.01699733336766561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,3584,0.01864853302637736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2560,0.015133866667747497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,2048,0.012376532951990763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1536,0.010484266281127929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,512,0.007247999807198842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,768,0.008414933085441589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,1024,0.009224533041318258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,256,0.006351999938488007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1536,128,0.0058794667323430385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,12288,0.05289599895477295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,10240,0.040099199612935385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,8192,0.03290133277575175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,16384,0.06673386891682943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,6144,0.026078933477401735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,7168,0.028819199403127032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,5120,0.021655466159184775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,4096,0.017828265825907387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,65536,0.2789333343505859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3584,0.015612799922625223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,3072,0.014492799838383993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2560,0.012676266829172769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,2048,0.010826667149861652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1536,0.009322667121887207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,256,0.0057536001006762184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,512,0.0063296000162760425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,768,0.007119999825954437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,1024,0.008087466657161712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,1024,128,0.00535999983549118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,12288,0.04484800100326538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,10240,0.036297599474589035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,8192,0.03012160062789917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,16384,0.05719253222147623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,6144,0.023215999205907188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,7168,0.02627840042114258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,4096,0.01696853240331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,5120,0.020153599977493285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,65536,0.2458709398905436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3072,0.01350826621055603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,3584,0.015218133727709452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2560,0.012202666203180949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,2048,0.01067733367284139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1536,0.009107200304667155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,768,0.007080533107121785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,1024,0.008032000064849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,128,0.005355733136336008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,512,0.006270933151245117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,768,256,0.005716266731421152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,12288,0.04076053301493327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,10240,0.03576960166295369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,8192,0.028075732787450153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,16384,0.054782934983571376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,6144,0.022282665967941283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,7168,0.025144533316294355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,4096,0.015827199816703795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,5120,0.018822399775187175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,65536,0.2255253314971924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3072,0.013082666198412576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2560,0.01176106631755829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,3584,0.014358400305112203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,2048,0.010140800476074218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1536,0.008707200487454731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,256,0.005622399846712748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,512,0.00613013356924057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,768,0.00687360018491745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,1024,0.0078015998005867004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,512,128,0.005163733164469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,10240,0.031091199318567915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,8192,0.024132267634073893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,12288,0.035936001936594644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,16384,0.04654826720555623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,6144,0.018971733252207437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,7168,0.021975467602411904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,4096,0.014424533645311991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,5120,0.01693013310432434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,65536,0.17407253583272297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3072,0.012162133057912191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,3584,0.01332373321056366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2048,0.009241599837938945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1536,0.008462933699289958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,2560,0.011044266819953918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,1024,0.00726506660381953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,128,0.004901333153247834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,768,0.006539733211199443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,256,0.005257600049177805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,256,512,0.005842133363087972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,10240,0.02879253427187602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,8192,0.023966934283574423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,12288,0.03466453154881795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,16384,0.046057601769765213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,5120,0.01644053359826406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,7168,0.021385600169499717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,6144,0.01924053430557251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,4096,0.014145066340764364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,65536,0.1702357292175293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2048,0.009057066837946574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3584,0.013058132926623025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,3072,0.011751466989517212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1536,0.008518399794896443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,2560,0.010916266838709514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,256,0.005221333106358846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,512,0.005718400080998739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,1024,0.0071839998165766404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,768,0.006422399977842967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,768,128,128,0.004871466755867004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,7168,0.4757685343424479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,8192,0.5429941177368164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,10240,0.7052672068277995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,12288,0.8519360224405924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,5120,0.34962774912516276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,6144,0.4145557403564453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,4096,0.27882986068725585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,16384,1.1012191772460938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3584,0.2449002742767334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1536,0.1166431983311971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,3072,0.21266454060872397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2048,0.1401898701985677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,2560,0.18068480491638184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,768,0.08012906710306802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,1024,0.08587093353271484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,128,0.05225600004196167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,512,0.06354346672693888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,65536,256,0.05454613367716471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,12288,0.20649919509887696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,8192,0.13365972836812337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,10240,0.18118826548258465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,16384,0.2695690790812174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,6144,0.10452053546905518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,5120,0.08741546471913655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,7168,0.11783786614735921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,4096,0.07068586349487305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3072,0.053729065259297694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2560,0.048089599609375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,3584,0.06296639839808146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,2048,0.03997226556142171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1536,0.03352533181508382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,768,0.02395306626955668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,1024,0.026428800821304322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,512,0.020179200172424316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,256,0.017520000537236534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,128,0.01593706707159678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,16384,65536,1.1042485555013022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,12288,0.16000320116678873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,10240,0.137664000193278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,16384,0.205894406636556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,7168,0.09056639671325684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,5120,0.06651306549708048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,6144,0.07927573521931966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,8192,0.10385493437449138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2560,0.03729386727015178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,4096,0.05383466482162476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3584,0.04903466701507568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,3072,0.041230932871500654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,65536,0.845088005065918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,2048,0.031386667490005495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1536,0.02649173339207967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,768,0.01887680093447367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,1024,0.02113386591275533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,512,0.016476800044377647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,256,0.013822933038075766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,12288,128,0.01302826702594757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,10240,0.11992639700571697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,12288,0.14915520350138348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,8192,0.09321066538492838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,16384,0.17335573832194012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,6144,0.0707797368367513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,7168,0.08012693723042806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,4096,0.049737600485483806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,5120,0.05995093186696371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3072,0.037298134962717694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,65536,0.768610127766927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2560,0.033913600444793704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,3584,0.043900799751281736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,2048,0.028189865748087566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1536,0.023336533705393472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,256,0.011442133784294128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,512,0.014154666662216186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,768,0.01625920037428538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,1024,0.01984106699625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,10240,128,0.010537599523862202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,10240,0.09508372942606608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,12288,0.11068693002065022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,16384,0.1392266591389974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,8192,0.06858346462249756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,7168,0.060779734452565515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,6144,0.05585493246714274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,5120,0.046673067410786945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,4096,0.03804479837417603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3584,0.034297601381937666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,3072,0.029901866118113202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2560,0.026014933983484905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,2048,0.022616533438364665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1536,0.019321600596110024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,1024,0.015970133741696677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,768,0.014467199643452963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,512,0.012250666817029316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,65536,0.5867178599039714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,128,0.010109866658846538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,8192,256,0.010506666700045268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,8192,0.06708587010701497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,10240,0.07928106784820557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,16384,0.12422186533610027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,12288,0.09572906494140625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,7168,0.0562613328297933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,4096,0.035478401184082034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,6144,0.048766934871673585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,5120,0.04404906829198201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3584,0.03261973261833191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,3072,0.027313067515691118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2560,0.024728532632191977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,2048,0.02176533341407776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1536,0.017249067624409996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,1024,0.014867200454076131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,768,0.012185600399971009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,512,0.01018773317337036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,65536,0.4916277249654134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,256,0.008675199747085572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,7168,128,0.007942399879296621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,8192,0.05945813258488973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,10240,0.08293333053588867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,16384,0.11861440340677898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,12288,0.09408640066782634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,7168,0.050619733333587644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,5120,0.038022398948669434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,4096,0.03202133377393086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,6144,0.0472821315129598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2048,0.01916266679763794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3584,0.028834132353464763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,3072,0.02456106742223104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,2560,0.02235520084698995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1536,0.016104533274968465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,1024,0.013914666573206582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,768,0.0116757333278656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,512,0.009796266754468281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,256,0.008692266543706258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,128,0.007911466558774312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,6144,65536,0.5165685335795085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,12288,0.08689280351003012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,8192,0.05237226486206055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,10240,0.06676692962646484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,16384,0.11442986329396565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,6144,0.042269865671793624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,7168,0.047005867958068846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,4096,0.02893226742744446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,5120,0.03506346543629964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,65536,0.47390079498291016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3584,0.026504532496134443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,3072,0.022111999988555908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2560,0.020476800203323365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,2048,0.017592533429463705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1536,0.014838400483131408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,256,0.00783253312110901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,512,0.00879039963086446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,768,0.01053439974784851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,1024,0.012618666887283326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,5120,128,0.007301333546638489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,10240,0.0530133326848348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,12288,0.06013226509094238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,16384,0.07714347044626871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,8192,0.03774506648381551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,7168,0.03522239923477173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,5120,0.025550933678944905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,6144,0.031205334266026813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,4096,0.022115200757980347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,65536,0.3179658571879069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2560,0.016103466351826988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3584,0.020643200476964316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,3072,0.017809067169825235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,2048,0.0143477330605189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1536,0.01284160017967224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,768,0.009064533313115438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,512,0.008255999783674877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,1024,0.010292266805966694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,256,0.007530666887760162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,4096,128,0.007169066866238911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,12288,0.05267200072606405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,10240,0.04512106577555339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,8192,0.03922559817632039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,16384,0.06924266815185547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,6144,0.029227733612060547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,7168,0.032995200157165526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,5120,0.026065067450205488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,4096,0.02116159995396932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,65536,0.26977599461873375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3072,0.01697173317273458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,3584,0.020350933074951172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1536,0.011872000495592753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2560,0.015524267156918844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,2048,0.013954133788744608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,256,0.006594133377075195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,512,0.007396266857783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,768,0.008265600105126699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,1024,0.00941973328590393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3584,128,0.0062165334820747375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,10240,0.042174931367238364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,12288,0.048401065667470294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,8192,0.03414186636606852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,16384,0.06424533526102702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,6144,0.02717546621958415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,7168,0.029449599981307983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,4096,0.0196288009484609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,65536,0.27991466522216796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,5120,0.02330986658732096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3072,0.015660799543062844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,3584,0.01841813325881958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1536,0.01088533302148183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2560,0.014593066771825156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,2048,0.012786133090655008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,768,0.0079434668024381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,256,0.00664106657107671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,1024,0.008983467022577922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,128,0.006108800073464712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,3072,512,0.00727040022611618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,12288,0.045102934042612716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,10240,0.03938560088475545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,8192,0.03156053423881531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,16384,0.06475946505864462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,7168,0.02727786699930827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,6144,0.025964800516764325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,65536,0.2541738669077555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,5120,0.021613866090774536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,4096,0.018373332420984902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3072,0.014679466684659323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,3584,0.017066667477289833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1536,0.009872000416119893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2560,0.013620266318321228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,2048,0.011864533027013142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,1024,0.008430932958920796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,256,0.006154666841030121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,128,0.005676800012588501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,768,0.007400533556938172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2560,512,0.00681386689345042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,8192,0.031344000498453775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,10240,0.04123306671778361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,12288,0.04779200156529744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,16384,0.06000533501307169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,6144,0.025120000044504803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,5120,0.021163733800252278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,7168,0.02701653242111206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,4096,0.017899733781814576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,65536,0.25703679720560707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3072,0.014315733313560485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2048,0.010769066214561463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,3584,0.016976000865300496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,2560,0.012776533762613932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1536,0.009635200103123982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,256,0.006356266637643178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,512,0.006925866504510244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,768,0.00732479989528656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,1024,0.008014933268229166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,2048,128,0.0059008002281188965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,12288,0.03943253358205159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,16384,0.049969065189361575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,10240,0.032414933045705155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,8192,0.026756266752878826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,7168,0.023566933472951253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,6144,0.020778665939966835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,4096,0.015564800302187601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,65536,0.22177386283874512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,5120,0.018458666404088338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3072,0.01227839986483256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,3584,0.01497066617012024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1536,0.008616532882054646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2560,0.011152000228563944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,2048,0.009751466910044353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,256,0.005636266867319743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,768,0.0067093332608540845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,512,0.006152533491452535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,128,0.005276800195376078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1536,1024,0.007517866790294647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,16384,0.04290453195571899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,8192,0.02262293299039205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,10240,0.03314346671104431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,12288,0.03822186787923177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,65536,0.19540586471557617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,6144,0.019800533850987755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,7168,0.020722132921218873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,5120,0.01649493376413981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,4096,0.013989333311716715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3072,0.011037866274515789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2048,0.008894933263460796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,2560,0.010597333312034607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,3584,0.013329066832860312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1536,0.008450133601824443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,256,0.005487999816735586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,768,0.0064533332983652755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,512,0.006054399907588959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,1024,0.007075199981530507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,1024,128,0.005165866514046987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,12288,0.03325440088907878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,10240,0.028802132606506346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,8192,0.024132267634073893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,16384,0.04474026759465535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,7168,0.021663999557495116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,6144,0.019052799542744955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,65536,0.18501653671264648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,4096,0.013585066795349121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,5120,0.01658986707528432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3072,0.010998400052388509
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,3584,0.013351466258366904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2560,0.010300800204277039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,2048,0.009144533673922222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1536,0.008091733356316884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,768,0.006414933502674103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,128,0.005116799970467886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,512,0.005990399916966756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,1024,0.00721919983625412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,768,256,0.005505066613356272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,8192,0.021244800090789794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,10240,0.02578666607538859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,12288,0.030979200204213457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,16384,0.03808533350626628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,65536,0.15684159596761066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,7168,0.019013333320617675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,5120,0.015191466609636942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,6144,0.01673706571261088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,4096,0.012409599622090657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2048,0.008377599716186523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3072,0.01053439974784851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,2560,0.009943466385205586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,3584,0.012377599875132244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1536,0.008221866687138875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,1024,0.006975999971230824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,256,0.005319466690222422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,512,0.005806933343410492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,768,0.006189866860707601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,512,128,0.0049781332413355505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,10240,0.02412266731262207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,12288,0.028750934203465778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,16384,0.03875840107599894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,8192,0.020248534282048543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,6144,0.016157866517702738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3584,0.012056533495585125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,7168,0.018311466773351034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,5120,0.014310399691263834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,65536,0.1495402654012044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,4096,0.012309333682060242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,3072,0.010222933689753215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2560,0.009738666812578838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,2048,0.008281599978605907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1024,0.006738133231798808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,1536,0.007926400005817413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,768,0.005983999868233999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,512,0.005565866827964783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,256,0.005082666873931885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,256,128,0.004839466512203216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,16384,0.03648746808369954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,8192,0.020114133755366005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,10240,0.02408426602681478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,12288,0.028118399779001872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,6144,0.01609813372294108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,65536,0.13701333999633789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,7168,0.01820053259531657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,5120,0.01418239971001943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,4096,0.01204373339811961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3584,0.011907200018564861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,3072,0.01011840005715688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2560,0.009566932916641235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,2048,0.008088533580303193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1536,0.007946666578451793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,1024,0.006702933212121327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,768,0.005922133227189382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,256,0.005021866659323374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,512,0.005543466905752818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,512,128,128,0.004709333181381226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,8192,0.3935797373453776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,7168,0.35099201202392577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,10240,0.5338165283203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,12288,0.620798937479655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,6144,0.30891520182291665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,5120,0.2591125329335531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,4096,0.20350079536437987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,16384,0.7934378941853841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,3584,0.1755082607269287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,1024,0.06737813154856363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,3072,0.15561706225077312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,2048,0.10951680342356365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,1536,0.0918346643447876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,2560,0.13440532684326173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,512,0.05014079809188843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,128,0.04071466525395711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,768,0.06296319961547851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,65536,256,0.043185067176818845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,8192,0.09793279965718588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,12288,0.15107839902242023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,10240,0.1302623987197876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,16384,0.1991221268971761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,4096,0.05346133311589559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,7168,0.08728746573130289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,6144,0.07748160362243653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,5120,0.064955735206604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,3584,0.04777173201243083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,2560,0.036449066797892254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,3072,0.041698133945465087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,2048,0.032304000854492185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,1536,0.027214932441711425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,1024,0.02127893368403117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,256,0.014418133099873862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,768,0.01949119965235392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,512,0.016516266266504924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,128,0.013030399878819784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,12288,0.13100159962972005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,10240,0.10850880146026612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,16384,65536,0.8478176116943359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,16384,0.15844160715738934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,6144,0.06698453426361084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,7168,0.07194133599599203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,8192,0.08298666477203369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,5120,0.05409706830978393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,3072,0.0335647980372111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,2560,0.030357333024342854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,4096,0.04287679990132649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,3584,0.03899626731872559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,65536,0.7213845570882161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,1536,0.02106346686681112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,768,0.014916266997655234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,1024,0.0178656001885732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,2048,0.026254934072494508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,512,0.013365333278973898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,256,0.010806399583816528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,12288,128,0.009789866209030152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,12288,0.10237226486206055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,10240,0.08659199873606363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,16384,0.13065280119578043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,8192,0.07091733614603678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,7168,0.06214400132497152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,5120,0.04632960160573323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,6144,0.052589865525563564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,4096,0.03688426812489827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,2560,0.02624959945678711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,3072,0.028782933950424194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,3584,0.03475519816080729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,2048,0.023221333821614586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,768,0.013727999726931252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,1024,0.015599999825159708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,1536,0.018747733036677042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,512,0.011809066931406657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,256,0.009692800045013428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,128,0.008959999680519104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,10240,65536,0.5116949399312337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,16384,0.1146399974822998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,12288,0.08968213399251303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,10240,0.07294613520304362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,8192,0.05903146664301554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,7168,0.04967360099156697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,6144,0.04467946688334147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,4096,0.031125332911809283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,5120,0.03697813351949056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,65536,0.4926485379536946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,3072,0.02458239992459615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,3584,0.0276202658812205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,2560,0.02259413401285807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,2048,0.01945599913597107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,1536,0.016266666849454246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,1024,0.01393280029296875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,768,0.011960533261299134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,512,0.010152533650398254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,256,0.008887466788291932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,8192,128,0.008160000046094257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,12288,0.07974932988484701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,10240,0.06526933511098226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,16384,0.10358613332112629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,8192,0.05525973240534464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,6144,0.04064106543858846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,7168,0.04713066816329956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,4096,0.029412267605463664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,5120,0.036150399843851724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,65536,0.4501759847005208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,3072,0.023605332771937052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,3584,0.02714560031890869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,2560,0.021514666080474854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,2048,0.019058134158452353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,1536,0.016130133469899496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,512,0.009962667028109233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,768,0.011685333649317424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,1024,0.013414399822553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,256,0.008526933193206788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,7168,128,0.007890133559703827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,12288,0.07753492991129557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,16384,0.10084160168965657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,10240,0.06518826484680176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,8192,0.05034453471501669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,6144,0.03874133427937825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,7168,0.04387413263320923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,5120,0.03257813254992167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,4096,0.02775893410046895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,65536,0.43367573420206706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,3072,0.02141439914703369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,3584,0.024626133839289348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,2560,0.01958720088005066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,2048,0.0174453337987264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,1536,0.014495999614397685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,512,0.009146666526794434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,768,0.010756267110506694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,1024,0.01253973344961802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,256,0.008070399860541026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,6144,128,0.007563733557860057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,10240,0.04681280056635539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,12288,0.05423893531163534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,16384,0.07085440158843995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,8192,0.03942506710688273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,6144,0.029948800802230835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,3584,0.020589866240819297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,5120,0.026601600646972656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,7168,0.034074668089548746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,4096,0.021755733092625937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,2048,0.014755200346310934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,3072,0.018054399887720743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,2560,0.016215466459592185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,1536,0.01258026659488678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,768,0.008861866593360902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,1024,0.010032000144322713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,65536,0.2745439847310384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,512,0.007918933530648549
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,256,0.007157333195209503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,5120,128,0.006671999891599019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,16384,0.06121386686960856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,10240,0.04224319855372111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,12288,0.048613333702087404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,8192,0.032357333103815715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,7168,0.028834132353464763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,5120,0.022261333465576173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,6144,0.02632853388786316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,4096,0.019386667013168334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,65536,0.26567467053731286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,3072,0.015518933534622192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,2560,0.014140799641609192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,3584,0.01706026593844096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,2048,0.013011200229326883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,1536,0.011055999994277954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,1024,0.008776533603668212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,768,0.007991466422875721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,512,0.007315200070540111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,256,0.006651733318964641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,4096,128,0.006151466568311056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,10240,0.03802773157755534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,12288,0.043450665473937986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,16384,0.05746560096740723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,8192,0.032041599353154496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,7168,0.027395200729370118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,5120,0.022227199872334798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,6144,0.024256000916163124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,4096,0.018105600277582803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,2560,0.013751467068990072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,2048,0.0127893328666687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,65536,0.236407470703125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,3584,0.017273600896199545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,3072,0.015236266454060874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,1536,0.01090666651725769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,1024,0.008690133690834045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,256,0.006444799900054932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,768,0.008088533580303193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,512,0.007354666789372762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3584,128,0.006055466830730438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,16384,0.05462506612141928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,10240,0.03732053438822429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,12288,0.04074133237202962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,8192,0.028945066531499225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,7168,0.025260800123214723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,6144,0.023375999927520753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,5120,0.020105600357055664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,4096,0.016703999042510985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,65536,0.23653012911478677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,3584,0.015986133615175882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,3072,0.014362667004267374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,2560,0.012825600306193032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,2048,0.011844266454378765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,1536,0.009716266393661499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,1024,0.00867733359336853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,768,0.007701333363850911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,512,0.007064533233642578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,256,0.006260266900062561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,3072,128,0.005830400188763936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,12288,0.04125973383585612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,16384,0.05393706560134888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,10240,0.035266133149464925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,8192,0.028819199403127032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,7168,0.024809600909550984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,6144,0.02272746761639913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,65536,0.22179946899414063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,5120,0.019735467433929444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,4096,0.01667840083440145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,3072,0.013942399621009826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,3584,0.014884266257286071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,2048,0.010877866546312969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,2560,0.012601600090662638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,1536,0.009499733646710713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,256,0.006247466802597046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,1024,0.007692799965540569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,768,0.007225599884986877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,512,0.006769066552321117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2560,128,0.005866666634877523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,12288,0.03551040093104045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,10240,0.03087573250134786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,8192,0.025602134068806966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,16384,0.04599253336588542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,7168,0.02225173314412435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,6144,0.020188800493876138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,65536,0.2000490665435791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,5120,0.01773973306020101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,4096,0.014979199568430582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,3584,0.013942399621009826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,3072,0.012454400459925335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,2560,0.011250133315722149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,1536,0.008449066678682964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,2048,0.009852799773216247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,768,0.006559999783833821
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,256,0.0056426664193471275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,128,0.0053151999910672505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,512,0.006137600044409434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,2048,1024,0.007150933146476746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,12288,0.03356800079345703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,10240,0.02868693272272746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,16384,0.044061867396036784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,8192,0.024195200204849242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,7168,0.021283199389775596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,6144,0.018962132930755615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,65536,0.1812714735666911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,4096,0.014470400412877402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,5120,0.017249067624409996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,3072,0.01142080028851827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,2560,0.010388267040252686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,1536,0.008227199812730153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,3584,0.01328213314215342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,2048,0.009548800190289815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,256,0.005490133166313171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,512,0.006113066772619883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,128,0.005273599922657013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,1024,0.007124266525109608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1536,768,0.006548266609509785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,12288,0.030616533756256104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,10240,0.02649173339207967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,8192,0.022166399161020915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,16384,0.039979732036590575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,65536,0.1618805408477783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,6144,0.01803306738535563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,7168,0.01981653372446696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,4096,0.012973866860071816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,5120,0.015837867061297098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,3072,0.010706133643786113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,3584,0.012098133563995361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,1536,0.0078015998005867004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,2560,0.009777067104975383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,2048,0.009078400333722432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,1024,0.0068906664848327635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,256,0.005410133302211762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,768,0.0063391998410224915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,512,0.005955199897289276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,1024,128,0.005131733417510986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,12288,0.029372799396514892
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,8192,0.02223680019378662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,10240,0.02601066629091899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,16384,0.04024853308995564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,5120,0.01552959978580475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,7168,0.01922453244527181
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,6144,0.01697493394215902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,4096,0.012498133381207784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,65536,0.15237332979838053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,3584,0.011953066786130269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,3072,0.01034879982471466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,2048,0.008845866719881693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,2560,0.009571199615796406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,1536,0.007639466722806294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,1024,0.006611200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,256,0.005248000224431356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,768,0.0061471998691558834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,128,0.004956800242265066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,768,512,0.005807999769846598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,12288,0.02629973292350769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,16384,0.03357866605122884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,8192,0.01917440096537272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,10240,0.02280319929122925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,7168,0.017232000827789307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,65536,0.13098560174306234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,6144,0.015241600076357522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,4096,0.011322666207949321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,5120,0.013275733590126038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,3584,0.011057066917419433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,3072,0.009795199831326802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,2560,0.009236266215642292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,2048,0.008148266871770223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,1536,0.007608533402283986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,256,0.005134933193524678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,1024,0.006386133531729381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,768,0.005960533519585928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,512,0.005603200197219849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,512,128,0.004866133133570353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,12288,0.025814400116602583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,10240,0.02172693411509196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,16384,0.03306879997253418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,8192,0.01829013427098592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,7168,0.016503467162450155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,6144,0.014896000425020853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,65536,0.12524373531341554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,4096,0.011252267162005107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,5120,0.01320319970448812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,3584,0.010957866907119751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,3072,0.00962773362795512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,2560,0.00902933379014333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,2048,0.008098133405049642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,1536,0.0075103998184204105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,1024,0.0062496001521746315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,512,0.005635199944178263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,768,0.005857066810131073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,256,0.005031466484069824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,256,128,0.004750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,12288,0.025276799996693928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,10240,0.021655466159184775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,16384,0.03264853358268738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,7168,0.01644373337427775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,8192,0.018247467279434205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,65536,0.12141226927439372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,5120,0.01318933367729187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,3584,0.01088106632232666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,4096,0.011152000228563944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,6144,0.014820266763369241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,3072,0.009530666470527648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,1536,0.007523199915885926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,1024,0.006214400132497152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,2560,0.008892800410588582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,512,0.005443199972311656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,2048,0.007977599898974102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,768,0.005780266722043356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,256,0.0049333333969116214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,384,128,128,0.004770133395989736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,7168,0.23245439529418946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,8192,0.2590378602345785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,10240,0.3518122673034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,12288,0.40853331883748367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,5120,0.17160746256510417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,16384,0.545253308614095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,6144,0.20378026962280274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,4096,0.14157652854919434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3584,0.12748479843139648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1536,0.06550079981486003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,1024,0.049951998392740886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2048,0.08051093419392905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,2560,0.09999893506368002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,256,0.031037867069244385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,3072,0.10881919860839843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,512,0.03591466744740804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,128,0.029009066025416058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,65536,768,0.046393601099650066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,10240,0.09359359741210938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,8192,0.07010133266448974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,12288,0.10447146892547607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,16384,0.13098986943562824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,6144,0.05506773392359415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,7168,0.06381653149922689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,5120,0.047437866528828934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,4096,0.039529601732889816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2560,0.028146133820215864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1536,0.020874667167663574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3072,0.032305065790812174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,3584,0.035946667194366455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,2048,0.02510400017102559
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,768,0.015450666348139444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,512,0.01300159990787506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,1024,0.01632960041364034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,256,0.01097706655661265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,128,0.010140800476074218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,12288,0.08960426648457845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,10240,0.07627306779225668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,16384,0.11144426663716633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,16384,65536,0.5559989293416341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,8192,0.057374934355417885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,5120,0.04091413418451945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,6144,0.044854398568471274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,7168,0.051958398024241126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,4096,0.03188266754150391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2048,0.02000746726989746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,2560,0.024074665705362954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3072,0.025867732365926106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,3584,0.029054933786392213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1024,0.014276267091433207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,1536,0.016131200393040977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,768,0.012074666221936543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,512,0.01025279959042867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,256,0.008564266562461852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,128,0.007886933286984761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,12288,65536,0.4845994631449382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,12288,0.0806709369023641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,16384,0.09958720207214355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,10240,0.06911146640777588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,8192,0.05480213165283203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,6144,0.039902933438618976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,7168,0.046233598391215006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,5120,0.034474666913350424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,4096,0.028604799509048463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3072,0.023341866334279378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,65536,0.4405365308125814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,3584,0.026298666000366212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2560,0.021211733420689903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,2048,0.018178133169809978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1536,0.014784000317255654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,512,0.00918826659520467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,768,0.011035733421643575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,1024,0.013128532965977987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,256,0.00768746683994929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,10240,128,0.007175466914971669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,12288,0.05608319838841756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,10240,0.047365331649780275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,16384,0.0723743995030721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,8192,0.039289601643880206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,7168,0.035572266578674315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,6144,0.031191466252009074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,5120,0.026910932858784993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,4096,0.023460266987482707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3584,0.021359999974568687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,3072,0.019323732455571493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2560,0.017033600807189943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,2048,0.015474133690198264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1536,0.012988799810409546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,1024,0.01116373340288798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,768,0.010005333026250203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,65536,0.29139200846354163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,512,0.008167466521263123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,256,0.007490133245786031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,8192,128,0.007234133283297221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,10240,0.04350933233896891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,12288,0.05143466790517172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,16384,0.0655893325805664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,8192,0.03626879851023356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,6144,0.02849919994672139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,7168,0.03260053396224975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,5120,0.024782933791478477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,4096,0.02100693384806315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2560,0.015411200126012168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3584,0.019432532787322997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,3072,0.017245866854985557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1536,0.011585066715876263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,2048,0.014004266262054444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,512,0.007225599884986877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,768,0.008346666892369587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,1024,0.00978559950987498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,65536,0.25649706522623694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,256,0.006549333532651265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,7168,128,0.006171733140945435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,16384,0.0602837324142456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,12288,0.046397864818573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,10240,0.040433065096537275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,8192,0.032509867350260416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,5120,0.02257279952367147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,6144,0.02600533366203308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,4096,0.01925440033276876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,7168,0.029883732398351032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3072,0.01583253343900045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,3584,0.01772159934043884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2560,0.014351999759674073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,2048,0.013141333063443502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,65536,0.252787192662557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1536,0.010988799730936687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,1024,0.009143466750780743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,768,0.007770666480064392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,512,0.007095466554164887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,256,0.006506666541099548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,6144,128,0.006132266422112783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,10240,0.036445868015289304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,12288,0.042430933316548666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,8192,0.02954559922218323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,16384,0.053223466873168944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,6144,0.023356799284617105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,7168,0.026716800530751546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3584,0.015965867042541503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,5120,0.020003199577331543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,4096,0.017015467087427773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,3072,0.014334932963053385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2048,0.01181013286113739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,2560,0.01356160044670105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,65536,0.22610774040222167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1536,0.009883733590443929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,1024,0.008055466910203297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,768,0.007349333167076111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,512,0.006657066444555919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,256,0.006044800082842508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,5120,128,0.005646933118502299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,10240,0.03411306540171306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,12288,0.03898880084355672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,8192,0.026978133122126262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,16384,0.049345068136850995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,5120,0.01856213410695394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,7168,0.024324266115824382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,6144,0.021297067403793335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,65536,0.2145077387491862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,4096,0.016033066312472026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2048,0.011115733782450359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3072,0.013585066795349121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,3584,0.014613333344459533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,2560,0.012836266557375589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1536,0.00962773362795512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,128,0.00591786652803421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,768,0.007101866602897644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,1024,0.007685333490371704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,256,0.00624533345301946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,4096,512,0.006663466493288676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,8192,0.025265065828959148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,12288,0.03582613468170166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,10240,0.030716800689697267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,16384,0.04589866797129313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,6144,0.019661867618560792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,7168,0.022949333985646567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,5120,0.017414400974909462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,4096,0.01492586632569631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,65536,0.19258559544881185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3584,0.013937066992123923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,3072,0.012716799974441528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2560,0.012012799580891926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1536,0.008411733309427898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,2048,0.010498133301734925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,768,0.006757333377997081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,1024,0.007436800003051758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,256,0.005819733440876007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,128,0.0054954667886098225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3584,512,0.006281599899133046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,10240,0.02866986592610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,8192,0.024039467175801597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,12288,0.033319467306137086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,16384,0.04366720120112101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,6144,0.01848213275273641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,7168,0.021388800938924153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,5120,0.01652479966481527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,4096,0.014069333672523499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,65536,0.17992107073465985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3072,0.012034133076667786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,3584,0.013219199577967324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1536,0.007877333462238312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2560,0.011212799946467083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,2048,0.010009599725405376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,1024,0.0069567998250325514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,768,0.006489600241184235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,256,0.0055167997876803074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,128,0.005262933174769084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,3072,512,0.005915733178456625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,12288,0.031616000334421794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,10240,0.0271722674369812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,8192,0.022802132368087768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,16384,0.04172906478246053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,6144,0.01774186690648397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,7168,0.0204202671845754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,65536,0.15602347056070964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,5120,0.016108799974123636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,4096,0.013833600282669067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3072,0.011693867047627766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,3584,0.012984533111254373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2560,0.010925867160161336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1536,0.007753600180149078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,2048,0.00937066674232483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,1024,0.006974933544794719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,256,0.005512533088525137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,512,0.005957333246866862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,768,0.006427733103434245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2560,128,0.005256533126036326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,8192,0.019409066438674925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,10240,0.02609066764513652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,12288,0.028344533840815228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,16384,0.035631998380025225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,65536,0.15456639925638835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,7168,0.018219733238220216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,6144,0.01609386702378591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,5120,0.014268799622853597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,4096,0.01256106694539388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3072,0.010700800021489461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,3584,0.01207360029220581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2048,0.008331733445326488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,2560,0.009943466385205586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1536,0.007582933207352956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,768,0.006154666841030121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,512,0.005862399935722351
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,256,0.005334400137265523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,1024,0.006505600114663441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,2048,128,0.00517546683549881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,12288,0.028542933861414592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,16384,0.037178667386372884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,8192,0.02067306637763977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,10240,0.024410667022069295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,65536,0.14054932594299316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,6144,0.01623466710249583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,7168,0.01874026656150818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,4096,0.012452266613642375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,5120,0.014942933122316995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3584,0.011545600493748982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,3072,0.010657067100207012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2560,0.009439999858538311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1536,0.007272533575693766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,2048,0.008525866270065307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,768,0.0061152001221974695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,512,0.005730133255322774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,1024,0.006680533289909363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,128,0.005066666503747304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1536,256,0.005331199864546458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,12288,0.024199465910593666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,16384,0.030421332518259687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,10240,0.022367999951044718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,8192,0.017809067169825235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,65536,0.11728320121765137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,6144,0.014552533626556396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,7168,0.016457600394884746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,5120,0.012968533237775168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,4096,0.010678399602572124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3584,0.010426666339238484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2048,0.00792746643225352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,3072,0.009453866879145305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,2560,0.009304533402125042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1536,0.007321600119272869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,1024,0.006364800035953522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,256,0.005115733544031779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,768,0.005907199780146281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,128,0.0049898669123649595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,1024,512,0.005562666555245717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,16384,0.030253867308298748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,8192,0.01722773313522339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,10240,0.020572799444198608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,12288,0.02377600073814392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,65536,0.11569600105285645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,7168,0.01623679995536804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,6144,0.013658666610717773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,5120,0.01221440037091573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,4096,0.010334933797518413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3584,0.010308266679445902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,3072,0.009261866410573322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2048,0.007794133325417836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,2560,0.008990933497746784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1536,0.007147733370463054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,1024,0.006266666452089946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,768,0.005738666653633118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,512,0.005448533097902933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,256,0.005015466610590617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,768,128,0.004807466765244802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,8192,0.016667733589808144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,65536,0.11517333189646403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,10240,0.020155733823776244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,16384,0.030469334125518797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,12288,0.023363200823465984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,5120,0.01195093293984731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,4096,0.010315733154614766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,7168,0.015399466951688132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,6144,0.013345066706339517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3584,0.010227200388908387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2048,0.007670400043328603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,3072,0.009196799993515015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,2560,0.00886826713879903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1536,0.007107200225194295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,512,0.005310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,256,0.004951466619968414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,1024,0.006180266539255777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,128,0.004780800143877665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,512,768,0.005770666897296906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,16384,0.029255467653274535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,8192,0.016422399878501893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,12288,0.02252053419748942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,7168,0.015326933066050211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,10240,0.019525333245595296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,65536,0.10554880301157635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,6144,0.013339733084042868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3584,0.010115200281143188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,5120,0.011940266688664753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,3072,0.009157333771387737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,4096,0.010279466708501179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2560,0.008762666583061218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,2048,0.007663999994595845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1536,0.006984533369541168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,512,0.0053269331653912864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,768,0.005739733576774597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,1024,0.00613013356924057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,256,0.00490880012512207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,256,128,0.004713599880536398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,12288,0.02250880002975464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,16384,0.028889600435892743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,7168,0.015264000495274863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,65536,0.10552106698354084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,10240,0.01947093407313029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,8192,0.01639359990755717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,5120,0.011907200018564861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,6144,0.013306666413942972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,4096,0.010179199775060018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3584,0.010013866424560546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,3072,0.009124267101287841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2560,0.008643200000127155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,2048,0.007606400052706401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1024,0.006140799820423126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,512,0.005189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,256,0.004855466882387797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,1536,0.006958933174610138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,128,0.004735999802748362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,256,128,768,0.00565119981765747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,7168,0.3178400039672852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,8192,0.3647253354390462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,10240,0.43413333892822265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,12288,0.5178506533304851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,4096,0.17980052630106608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,6144,0.2713568051656087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,5120,0.22474452654520669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,16384,0.6830282847086588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3584,0.15570346514383954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,3072,0.1364693323771159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2560,0.11439359982808431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1536,0.06980906327565511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,2048,0.090720001856486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,512,0.03506133159001668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,256,0.02985386649767558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,768,0.04560106595357259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,1024,0.04864319960276286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,65536,128,0.02763413389523824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,8192,0.0964298645655314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,16384,0.17982932726542156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,10240,0.11550613244374593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,12288,0.13461653391520184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,7168,0.08421759605407715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,4096,0.04966613451639811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,5120,0.06116586526234945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,6144,0.0722869316736857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3584,0.043892268339792886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,3072,0.039773865540822344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1536,0.021893332401911415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2560,0.03399466673533122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,2048,0.027369600534439088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,256,0.010689066847165425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,1024,0.01676586667696635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,768,0.015031466881434122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,128,0.009716266393661499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,512,0.012939733266830445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,10240,0.1017301321029663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,12288,0.11634026368459065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,16384,65536,0.6871882756551106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,16384,0.1553877353668213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,6144,0.06486186583836874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,7168,0.07470293045043945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,8192,0.08730133374532065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,5120,0.054788267612457274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,4096,0.04460479815800984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3584,0.040321067969004316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,3072,0.035417600472768145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2560,0.02937493324279785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,2048,0.024514132738113405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,768,0.013198933005332947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1536,0.01963520050048828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,1024,0.014777599771817525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,512,0.010462933778762817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,256,0.008703999718030294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,128,0.00772159993648529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,12288,65536,0.5867114384969075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,12288,0.11024746894836426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,10240,0.09349973201751709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,8192,0.08084800243377685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,16384,0.14827413558959962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,5120,0.05094720125198364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,6144,0.06065706809361776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,4096,0.04158080021540324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,7168,0.07033812999725342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2048,0.023141332467397056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3584,0.03706560134887695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,3072,0.03221013347307841
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,2560,0.02770986755688985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1536,0.01832853356997172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,1024,0.013949867089589438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,768,0.011978666981061299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,512,0.009381332993507385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,256,0.00766186664501826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,65536,0.5472725550333659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,10240,128,0.007020799815654755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,12288,0.07358079751332601
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,10240,0.06282133261362711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,16384,0.0948970635732015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,8192,0.05023359855016073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,6144,0.039390933513641355
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,5120,0.03291626572608948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,7168,0.04474133253097534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,4096,0.02877119978268941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2560,0.019708800315856933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3072,0.022294400135676067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,3584,0.025727999210357667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1024,0.011496532956759136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,2048,0.01705706715583801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,768,0.010148266951243084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,65536,0.35735359191894533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,1536,0.014337066809336343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,512,0.008481066425641377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,256,0.007502933343251546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,8192,128,0.007008000214894612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,12288,0.06807786623636881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,10240,0.05733760197957357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,16384,0.0890666643778483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,8192,0.04835093418757121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,7168,0.04203946590423584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,6144,0.03646399974822998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,5120,0.031181865930557252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,4096,0.026837333043416338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3584,0.02377706567446391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2048,0.01592639982700348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,65536,0.33183253606160484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,2560,0.01825493375460307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1536,0.012842667102813721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,3072,0.021227733294169108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,1024,0.010532266894976298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,768,0.009001599748929341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,256,0.006491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,128,0.005992533266544342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,7168,512,0.007525333265463512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,10240,0.05399253368377686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,12288,0.06431039969126383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,16384,0.08366080125172934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,8192,0.04501760005950928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,6144,0.03424853483835856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,4096,0.02481173276901245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,7168,0.03943146864573161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,5120,0.029689600070317585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3584,0.023002666234970093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2560,0.017249067624409996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,3072,0.01994880040486654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,2048,0.015124266346295675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,768,0.00843946635723114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1536,0.012574932972590127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,1024,0.010007466872533162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,65536,0.30978666941324867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,512,0.007388799885908763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,256,0.00652159998814265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,6144,128,0.005964800218741099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,8192,0.04281599919001262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,16384,0.07836266358693442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,12288,0.058840533097585045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,10240,0.050964268048604336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,5120,0.02804800073305766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,4096,0.02355519930521647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,6144,0.03287360072135925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,7168,0.03735893170038859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3584,0.021528534094492593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,3072,0.018756266434987387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,65536,0.2815648078918457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2560,0.01642346680164337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,2048,0.013886933525403341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,768,0.008019199967384339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1536,0.011720533172289532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,512,0.006939733525117238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,256,0.006113066772619883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,128,0.00552106648683548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,5120,1024,0.009220266342163086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,12288,0.05308800141016642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,16384,0.06785919666290283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,8192,0.03686293363571167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,10240,0.044842668374379474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,7168,0.03302186727523804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,6144,0.029410133759180706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,4096,0.021444267034530638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,5120,0.024238934119542442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,65536,0.2635978698730469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3584,0.020201599597930907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,3072,0.01802240014076233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2048,0.012896000345547994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,2560,0.015188266833623251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1536,0.011317333579063416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,1024,0.008717866738637288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,768,0.007915733257929485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,512,0.007107200225194295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,256,0.006405333181222279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,4096,128,0.005899733304977417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,16384,0.07060053348541259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,8192,0.03761813243230184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,10240,0.046646400292714434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,12288,0.05519359906514486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,6144,0.029747200012207032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,4096,0.021413334210713706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,7168,0.033844268321990965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3584,0.01953386664390564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,5120,0.02564479907353719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,65536,0.2544650713602702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,3072,0.017237333456675212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2560,0.015064533551534018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,2048,0.012823466459910074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1536,0.010202667117118836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,1024,0.008433066805203756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,768,0.007389866809050243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,512,0.0065194666385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,256,0.005847466488679251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3584,128,0.005432533224423727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,12288,0.05312639872233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,10240,0.04443200031916301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,16384,0.06828266779581706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,8192,0.036735999584198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,7168,0.032902399698893234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,6144,0.028460800647735596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,5120,0.02418346603711446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,4096,0.020878932873407998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,65536,0.24258453051249185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3584,0.018959999084472656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,3072,0.01691306630770365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2560,0.014532267053922018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,2048,0.012285866340001424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1536,0.009969066580136616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,1024,0.008156799773375193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,768,0.0071274667978286745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,512,0.006237866481145223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,256,0.00565119981765747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,3072,128,0.00516480008761088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,12288,0.05207466681798299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,10240,0.0433077335357666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,8192,0.03582293192545573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,16384,0.06661440134048462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,7168,0.03241066733996074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,65536,0.23554347356160482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,5120,0.02347946763038635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,4096,0.02035520076751709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,6144,0.028486400842666626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3072,0.01649066706498464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,3584,0.018066134055455527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1024,0.007969066500663757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2560,0.013871999581654868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,2048,0.011451733112335206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,1536,0.009545600414276123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,256,0.005655466516812643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,768,0.00697920024394989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,512,0.006268799801667531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2560,128,0.005218133330345154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,16384,0.059329064687093105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,12288,0.046860798199971514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,10240,0.03988159894943237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,8192,0.03298239906628926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,7168,0.029630933205286664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,65536,0.21271680196126302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,6144,0.025075199206670125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,5120,0.021887999773025513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3072,0.015051733454068503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,4096,0.019058134158452353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,3584,0.017459199825922648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2560,0.012753066420555115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,2048,0.010745599865913391
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1536,0.009218133489290873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,1024,0.007558399935563405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,512,0.006190933287143707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,768,0.006914133330186208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,16384,0.06107519865036011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,256,0.005674666663010915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,2048,128,0.005177600185076395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,12288,0.04691093365351359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,10240,0.04011199871699016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,8192,0.03271893262863159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,7168,0.030134399731953938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,65536,0.20814933776855468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,6144,0.026026666164398193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,5120,0.02192853291829427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,4096,0.019053866465886436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3584,0.01649386684099833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,3072,0.014841600259145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2560,0.012739200393358865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,2048,0.010664533575375874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1536,0.008981333176294962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,1024,0.0075765331586201985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,512,0.006000000238418579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,768,0.006757333377997081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,256,0.005421866476535797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1536,128,0.005015466610590617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,12288,0.041553068161010745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,16384,0.05543359915415445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,10240,0.035743999481201175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,8192,0.029948800802230835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,65536,0.18794347445170084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,7168,0.026430932680765788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,6144,0.02331733306248983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,5120,0.019332265853881835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,4096,0.01686613361040751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3072,0.013898666699727377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,3584,0.01539413332939148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2560,0.01223466694355011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,768,0.006613333523273468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,2048,0.009969066580136616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1536,0.008959999680519104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,1024,0.007277866701285045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,512,0.005843199789524078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,256,0.005328000088532766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,1024,128,0.004915200173854828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,16384,0.0448309342066447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,12288,0.03491520086924235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,10240,0.030430932839711506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,65536,0.16711360613505047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,8192,0.024971733490626015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,7168,0.02213866710662842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3584,0.013204266627629599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,6144,0.01958400011062622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,5120,0.016823466618855795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,4096,0.014331733187039694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2048,0.008889599641164144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,3072,0.011495467027028401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1536,0.007987200220425924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,2560,0.010386133193969726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,1024,0.006617600222428639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,768,0.006102400024731954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,512,0.005650133391221364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,256,0.005226666728655497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,768,128,0.004866133133570353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,12288,0.02608319918314616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,16384,0.032798933982849124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,10240,0.022115200757980347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,65536,0.12436373233795166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,7168,0.0165173331896464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,6144,0.014525866508483887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,4096,0.011296000083287556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,8192,0.0183242658774058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,5120,0.01302186648050944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3584,0.010914132992426554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,3072,0.009305600325266521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2560,0.008821333448092144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1536,0.007210666437943776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,1024,0.006182399888833364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,768,0.005724800129731497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,2048,0.0076799998680750535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,512,0.005446400245030721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,128,0.004774400095144907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,512,256,0.005009066561857859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,65536,0.09734933376312256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,16384,0.027204267183939618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,8192,0.015482667088508605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,7168,0.014326399564743042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,12288,0.02136960029602051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,10240,0.018453333775202432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3072,0.008831999699274699
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,5120,0.011370666821797689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,6144,0.0127893328666687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,4096,0.010217600067456563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,3584,0.010039466619491576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2048,0.007226666808128357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,2560,0.008222933113574981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,768,0.005605333546797434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,512,0.005328000088532766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1536,0.006939733525117238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,1024,0.006049066781997681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,12288,0.021331199010213218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,256,0.004890666902065277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,256,128,0.0047658666968345646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,65536,0.09741866588592529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,16384,0.026877866188685103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,10240,0.01849386692047119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,8192,0.015468800067901611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,6144,0.012778666615486146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,4096,0.010204799969991048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,7168,0.014255999525388082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3584,0.009864532947540283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,5120,0.011383466919263204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,3072,0.008686932921409606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2048,0.007134933272997539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1024,0.0060362666845321655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,768,0.005504000186920166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,2560,0.008124800026416778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,1536,0.006971733272075653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,512,0.005195733408133189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,256,0.004866133133570353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,192,128,128,0.004676266511281332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,8192,0.40408531824747723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,7168,0.3531754811604818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,10240,0.47545601526896164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,12288,0.5416320164998372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,5120,0.25147199630737305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,4096,0.19775892893473307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,16384,0.7109696070353191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3584,0.17480640411376952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,6144,0.30420799255371095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1536,0.0775274674097697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2048,0.09997866948445638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,1024,0.05411946773529053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,3072,0.14790933926900227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,2560,0.1259050687154134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,512,0.034585599104563394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,768,0.045051733652750656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,256,0.029331199328104657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,65536,128,0.02686506708463033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,8192,0.10451412995656331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,12288,0.14484267234802245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,10240,0.12582293351491292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,16384,0.19229866663614908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,6144,0.08109119733174643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,5120,0.06619093418121338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,4096,0.05549973249435425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,7168,0.09183146953582763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3584,0.04929279883702596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,3072,0.0420576016108195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2048,0.0297760009765625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1536,0.023574399948120116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,2560,0.03646186590194702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,1024,0.01754666765530904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,768,0.015340800086657206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,128,0.009682133793830872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,512,0.013011200229326883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,256,0.0107232004404068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,10240,0.11090346972147624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,12288,0.12566826343536378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,16384,0.16359786987304686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,7168,0.08485226631164551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,16384,65536,0.7077184041341146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,6144,0.06904959678649902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,8192,0.09599680105845133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,4096,0.04938240051269531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3584,0.043508267402648924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,5120,0.061907200018564854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1536,0.020558933417002358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,3072,0.036195198694864913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2560,0.03313600023587544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,512,0.01143893301486969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,2048,0.026663466294606523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,256,0.008757332960764568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,1024,0.015774933497111003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,768,0.013215999801953635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,128,0.007768533130486806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,12288,65536,0.6233472188313802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,8192,0.08824000358581544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,12288,0.12225493590037029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,10240,0.1052064021428426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,16384,0.15487039883931478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,7168,0.07647039890289306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,6144,0.06774506568908692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,5120,0.05580799976984659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,4096,0.04641600052515666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3072,0.035778133074442546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,3584,0.041416533788045246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1536,0.019594667355219524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2560,0.030086400111516316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,1024,0.014939733346303306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,2048,0.024752000967661537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,768,0.012132267157236736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,512,0.00960213343302409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,256,0.007727999985218048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,128,0.006977066894372304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,10240,65536,0.5746016184488932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,16384,0.10092053413391114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,8192,0.054894932111104335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,10240,0.06754240194956461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,12288,0.07886293729146322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,6144,0.042971734205881754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,7168,0.04754453500111898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,4096,0.030051199595133464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3584,0.027536000808080035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,3072,0.02360746661822001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,5120,0.035411198933919266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2560,0.021051732699076335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,2048,0.017918932437896728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1536,0.014808533589045205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,512,0.0086325337489446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,1024,0.012063999970753986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,768,0.01027413308620453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,128,0.006984533369541168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,256,0.007541333138942718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,8192,65536,0.3675797462463379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,12288,0.07371520201365153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,10240,0.06313706636428833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,16384,0.09289920330047607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,7168,0.045025066534678145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,8192,0.051999998092651364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,6144,0.038696531454722086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,5120,0.03301333387692769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,4096,0.027923200527826948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3584,0.02483946681022644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2560,0.01935360034306844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,2048,0.016660267114639284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1536,0.013354667027791343
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,3072,0.021814399957656862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,256,0.006671999891599019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,1024,0.01055680016676585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,65536,0.34344959259033203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,768,0.008959999680519104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,512,0.007638399799664815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,7168,128,0.005936000247796377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,10240,0.059631999333699545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,12288,0.06925439834594727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,16384,0.09069546858469645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,8192,0.04731626510620117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,5120,0.031652265787124635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,6144,0.03686720132827759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,7168,0.04168426593144735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,4096,0.02653866608937581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3584,0.023819732666015624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,3072,0.02093120018641154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1536,0.012893866499265036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2560,0.018551466862360637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,2048,0.015546666582425437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,512,0.0075647999842961625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,1024,0.010683733224868774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,768,0.008531199892361958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,256,0.0066336000959078475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,128,0.005906133353710175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,6144,65536,0.3176213264465332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,10240,0.05635199944178263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,12288,0.06540373166402182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,16384,0.08445226351420085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,8192,0.044735999902089436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,5120,0.030057599147160847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,4096,0.02483733296394348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,7168,0.0397546648979187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,6144,0.03514986832936605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3584,0.02244480053583781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2560,0.01768640081087748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,3072,0.019409066438674925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,65536,0.3022826512654623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1536,0.012334932883580525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,1024,0.00965333382288615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,768,0.008109866579373678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,512,0.007044266661008198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,256,0.006099199752012888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,2048,0.01470186710357666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,5120,128,0.005530666808287303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,16384,0.07383039792378744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,10240,0.04990826845169068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,12288,0.057886934280395506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,8192,0.03986666599909465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,7168,0.03638293345769246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,6144,0.030873600641886396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,5120,0.026641066869099932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3072,0.017772799730300902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,4096,0.022317866484324135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,3584,0.02181866765022278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2048,0.013611732920010885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,65536,0.27610880533854165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,2560,0.016371200482050575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,768,0.008108800152937572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,512,0.007291733225186665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1536,0.011845333377520244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,1024,0.00921066701412201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,256,0.006389333307743073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,4096,128,0.005922133227189382
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,12288,0.06010026534398397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,16384,0.0792095979054769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,10240,0.05267839829126993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,8192,0.0408949335416158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,7168,0.03734933137893677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,4096,0.023090134064356484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,6144,0.031702399253845215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,5120,0.02858453392982483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3072,0.01816640098889669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,65536,0.26722561518351234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,3584,0.02148373325665792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2560,0.016294399897257485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,2048,0.013464533289273582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,512,0.006622933348019918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,256,0.0059232001503308615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1536,0.010803199807802836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,768,0.007619200150171916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,1024,0.008676266670227051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3584,128,0.005406933526198069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,12288,0.05811200141906738
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,16384,0.07334506511688232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,8192,0.04119786818822225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,10240,0.05019199848175049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,7168,0.03604160149892171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,4096,0.022514132658640544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3584,0.020821332931518555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,6144,0.031142399708429976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,5120,0.027926399310429888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2560,0.01581760048866272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,65536,0.26088852882385255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1536,0.010771200060844421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,3072,0.017762132485707603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,1024,0.008617599805196125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,768,0.007285333176453908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,2048,0.012837333480517068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,256,0.005745066702365876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,512,0.006390400230884552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,3072,128,0.005189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,12288,0.05548906723658244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,16384,0.07119359970092773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,10240,0.0490229328473409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,8192,0.04027626514434814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,5120,0.027149866024653118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,7168,0.03420480092366536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,65536,0.25138346354166663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,6144,0.030455466111501055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,4096,0.02257706721623739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3584,0.019703465700149535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,3072,0.017246933778127034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2560,0.015774933497111003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,2048,0.012494933605194092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1536,0.010308266679445902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,1024,0.008337066570917765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,768,0.007289599875609081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,512,0.00636053333679835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,256,0.005685333410898844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2560,128,0.00521066685517629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,12288,0.0512714664141337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,8192,0.036407466729482016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,10240,0.042393600940704344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,16384,0.06440000136693319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,5120,0.02291839917500814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,65536,0.22969172795613607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,7168,0.031023999055226643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,6144,0.0282858669757843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,4096,0.019604265689849854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2048,0.011672533551851908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1536,0.009742933511734008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,2560,0.014059733351071677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3584,0.01795519987742106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,3072,0.015528532862663268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,128,0.005156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,1024,0.0081045334537824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,768,0.00713919997215271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,512,0.006295466423034668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,2048,256,0.0057205334305763245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,16384,0.06814933617909749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,10240,0.04653333425521851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,8192,0.03670613368352254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,12288,0.052444799741109216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,5120,0.02474773327509562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,4096,0.02094399929046631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,65536,0.22992960611979166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,7168,0.03311786651611328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,6144,0.029768532514572142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2560,0.0139765332142512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3584,0.01830613414446513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,3072,0.016007467110951742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,2048,0.011832533280054729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1536,0.009998933474222819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,768,0.006980266670385997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,1024,0.007981866598129272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,512,0.006221866607666016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,16384,0.06026773452758789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,256,0.005492266515890757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1536,128,0.004961066444714864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,8192,0.03392639954884847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,7168,0.030316799879074097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,65536,0.20242026646931968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,12288,0.046590932210286456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,10240,0.0406389315923055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3584,0.016471466422080992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,6144,0.026268800099690754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,5120,0.02236586610476176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,4096,0.017833600441614784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,3072,0.014612266421318054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1024,0.007537066439787547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2560,0.0127402663230896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,2048,0.01107199986775716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,1536,0.009477333227793375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,768,0.006793599824110668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,256,0.005443199972311656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,512,0.0061749334136645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,1024,128,0.004984533290068308
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,12288,0.039930665493011476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,16384,0.051685333251953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,10240,0.03385173479715983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,65536,0.18781545956929524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,8192,0.027800534168879194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,6144,0.02168533404668172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,7168,0.02439893285433451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,5120,0.018286933501561485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,4096,0.01560640037059784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3584,0.014453333616256715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,3072,0.012537599603335062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2560,0.011352533102035522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1024,0.0070720002055168155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,768,0.0062282666563987735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,2048,0.009573333462079366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,1536,0.008406399687131246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,512,0.00580266664425532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,256,0.005329066514968872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,768,128,0.004890666902065277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,10240,0.02654613256454468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,8192,0.022175999482472737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,16384,0.04102506637573242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,65536,0.14201386769612628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,12288,0.0314517339070638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,6144,0.01776640017827352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,5120,0.015076266725858054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,4096,0.012611200412114462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,7168,0.018927999337514243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2560,0.009199999769528707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3072,0.009920000036557516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,2048,0.008121599753697712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,3584,0.011415466666221619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1024,0.006244266529877981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,768,0.005780266722043356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,256,0.005055999755859375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,1536,0.0075082664688428235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,128,0.00476800004641215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,512,512,0.005438933273156484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,16384,0.02718186577161153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,12288,0.020911999543507895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,8192,0.015154133240381876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,10240,0.017813332875569663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,65536,0.09563306967417398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,5120,0.011273599664370219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,7168,0.013778133193651834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,6144,0.0123989333709081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,4096,0.0100874662399292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3584,0.009753599762916565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2048,0.007166933516661327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,2560,0.008201600114504496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,3072,0.008449066678682964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1536,0.0069354668259620665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,1024,0.005957333246866862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,768,0.005674666663010915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,512,0.005285333096981049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,128,0.004690133531888326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,256,256,0.0049098665515581764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,16384,0.025893332560857137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,8192,0.015149866541226705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,7168,0.013738666971524557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,12288,0.020641066630681357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,10240,0.017799466848373413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,6144,0.012397866447766621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,65536,0.09411626656850179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,5120,0.011366400122642516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3584,0.00965226689974467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,4096,0.009924266735712688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,3072,0.008380800485610962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2560,0.008110933502515157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,2048,0.007095466554164887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1536,0.006970666845639547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,1024,0.005993600189685822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,256,0.004852266609668731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,512,0.005190399785836538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,768,0.005504000186920166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,160,128,128,0.004625066618124644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,8192,0.19030507405598956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,7168,0.16954026222229004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,10240,0.23604373931884765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,12288,0.2785546620686849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,5120,0.12659520308176678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,6144,0.14817172686258953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,16384,0.3636874516805013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3584,0.09632639884948731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,4096,0.10443733533223469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,3072,0.08229760328928629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2560,0.07102826436360678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1024,0.03561280171076457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,1536,0.04755200147628784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,2048,0.06510186592737834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,768,0.03148693243662516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,512,0.024666666984558105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,256,0.01857066750526428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,65536,128,0.016940800348917644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,8192,0.055194667975107824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,16384,0.10093013445536296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,10240,0.06629653374354044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,12288,0.07830719947814942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,7168,0.04953813155492147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,6144,0.04412906567255656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,4096,0.03258133331934611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,5120,0.0382421334584554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3072,0.02630079984664917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2048,0.020091732343037925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,2560,0.02322559952735901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1536,0.016174933314323424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,3584,0.0295360008875529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,768,0.011687466502189636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,512,0.00995306670665741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,1024,0.013102933764457703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,128,0.007158400118350982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,256,0.007589333256085714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,10240,0.05223146677017212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,12288,0.0612064003944397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,16384,0.07896640300750732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,16384,65536,0.37118399937947594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,8192,0.04360746542612712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,7168,0.038984533150990805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,6144,0.03473386764526367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,4096,0.025573333104451496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,5120,0.03343146642049154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2560,0.018257067600886027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3584,0.0233024001121521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,3072,0.02070186734199524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1536,0.012980266412099203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,2048,0.015506133437156677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,768,0.009643733501434326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,512,0.0073066666722297665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,1024,0.010687999924023946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,256,0.006484266618887584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,128,0.006100266675154368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,12288,65536,0.28973118464152015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,16384,0.06760853131612142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,8192,0.04188160101572673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,10240,0.04495253165562947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,12288,0.05279786586761474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,7168,0.03362026611963908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,6144,0.02998720010121663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,4096,0.02218986749649048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,5120,0.026235733429590864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3072,0.017644800742467246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2560,0.01576640009880066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,3584,0.020091732343037925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,2048,0.013542399803797404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1536,0.011554132898648579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,1024,0.009693866968154908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,512,0.006589866677920024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,128,0.005633066594600678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,65536,0.24611199696858727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,768,0.008429867029190064
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,10240,256,0.0060479998588562015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,8192,0.03187839984893799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,10240,0.038172801335652665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,12288,0.048884268601735434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,16384,0.05649066766103109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,5120,0.022320000330607097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,7168,0.028471465905507403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,6144,0.02542720039685567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,4096,0.01906986633936564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,65536,0.20875306129455568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3072,0.015340800086657206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,3584,0.01771519978841146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2048,0.012244266271591187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,2560,0.013962666193644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1024,0.008901333808898926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,1536,0.011635200182596842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,768,0.007437866429487865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,512,0.006651733318964641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,256,0.006172800064086914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,8192,128,0.005842133363087972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,12288,0.040830934047698976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,10240,0.034924801190694174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,16384,0.05200746854146322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,8192,0.029146667321523028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,6144,0.023520000775655112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,7168,0.0262773334980011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,5120,0.020334933201471964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,4096,0.017592533429463705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,65536,0.1882805347442627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3072,0.01397119959195455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,3584,0.01567466656366984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2560,0.012871467073758445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,2048,0.011148800452550251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1536,0.010565333565076192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,1024,0.0076447998483975725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,768,0.006750933329264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,512,0.006287999947865804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,256,0.005739733576774597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,7168,128,0.0055167997876803074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,12288,0.03673493464787801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,16384,0.04711893399556478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,10240,0.03142506678899129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,8192,0.026324266195297243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,7168,0.02363733251889547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,5120,0.018549333016077675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,6144,0.021271467208862305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,4096,0.015526400009791056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,65536,0.1683519999186198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3584,0.014283733566602072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,3072,0.012716799974441528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2560,0.011718400319417318
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,2048,0.010441600282986959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1536,0.009851732850074768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,768,0.0062720000743865965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,512,0.00584853341182073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,1024,0.007158400118350982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,128,0.005245866874853769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,6144,256,0.005498666564623515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,12288,0.033580799897511796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,10240,0.02909226616223653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,8192,0.02430400053660075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,16384,0.04315413236618042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,6144,0.01958720088005066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,7168,0.021897600094477335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,5120,0.016956800222396852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,4096,0.014451199769973755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,65536,0.15677226384480794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3584,0.013281066219011942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,3072,0.012000000476837159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2560,0.01120746632417043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,2048,0.009942400455474853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1536,0.008516266942024231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,768,0.006363733112812043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,1024,0.006852266689141591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,512,0.005892266829808554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,128,0.005322666466236114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,5120,256,0.005533866584300995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,8192,0.02031146685282389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,10240,0.02410879929860433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,12288,0.02828693389892578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,16384,0.03601599931716919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,6144,0.016568533579508462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,5120,0.01474453310171763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,7168,0.01812373399734497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,4096,0.0127402663230896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,65536,0.1340511957804362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3584,0.011817600329717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,3072,0.0108106662829717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2048,0.008906666437784832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,2560,0.010214400291442872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1536,0.008180266618728638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,1024,0.006361599763234456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,512,0.005771733323733012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,768,0.006088533500830332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,128,0.005143466591835022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,4096,256,0.005385600030422211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,8192,0.01908906698226929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,12288,0.026730666557947796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,10240,0.022826667626698813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,16384,0.034227200349171955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,7168,0.01755839983622233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,6144,0.01562879979610443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,5120,0.013933866222699483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,65536,0.12404906749725342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,4096,0.012283733487129212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3072,0.01039573351542155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,3584,0.011587199568748475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2560,0.00995306670665741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,2048,0.008357333143552144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1536,0.007483733197053273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,256,0.005322666466236114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,1024,0.006307200094064077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,768,0.005985066791375478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,128,0.005098666747411092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3584,512,0.005692799886067709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,10240,0.023450666666030885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,8192,0.01941546599070231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,12288,0.027344000339508057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,16384,0.03573973178863525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,6144,0.01589120030403137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,7168,0.017653334140777587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,65536,0.13173013528188068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,5120,0.014459733168284097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,4096,0.012664533654848733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3072,0.010440533359845478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,3584,0.011520000298817952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2560,0.009636267026265462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,2048,0.008225066463152568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1536,0.0071263998746871945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,512,0.005689600110054016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,768,0.006028800209363302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,256,0.005319466690222422
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,1024,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,3072,128,0.0051136001944541935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,12288,0.026146133740743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,10240,0.02262506683667501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,8192,0.018833067019780478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,16384,0.034119466940561935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,65536,0.12373332977294922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,6144,0.015454933047294617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,7168,0.017139200369517008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,5120,0.014178133010864258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,4096,0.012205866972605388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3072,0.009956266482671101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,3584,0.011246933539708456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2560,0.009011200070381165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1536,0.007038933535416921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,2048,0.008076799909273784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,512,0.005700266857941946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,768,0.0058442667126655575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,1024,0.006491733094056447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,256,0.005250133574008942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2560,128,0.0049898669123649595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,12288,0.021921066443125407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,10240,0.01920106609662374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,8192,0.016215466459592185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,16384,0.02866239945093791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,65536,0.10350613594055176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,7168,0.014762666821479798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,6144,0.0137472003698349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,5120,0.012575999895731608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,4096,0.010908800363540649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3584,0.010360532999038696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,3072,0.008813866972923278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2048,0.007337599992752075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,2560,0.00834986666838328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1536,0.007021866738796234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,1024,0.006155733267466227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,768,0.005735466877619425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,128,0.004897066454092661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,256,0.005118933320045471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,2048,512,0.005458133419354757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,12288,0.020920532941818237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,16384,0.027131734291712443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,10240,0.018665599822998046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,8192,0.01558080017566681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,65536,0.10258453687032063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,7168,0.014427733421325684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,5120,0.012083199620246888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,6144,0.013382400075594584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,4096,0.010156800349553425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3584,0.009764267007509868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,3072,0.008390399813652038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2048,0.007166933516661327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,2560,0.008190933366616566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1536,0.007017600039641063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,1024,0.0060245335102081295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,768,0.005640533566474914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,256,0.0050453335046768185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,512,0.005411200225353241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1536,128,0.0048320000370343525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,12288,0.020732800165812172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,16384,0.025962666670481367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,10240,0.018305067221323648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,8192,0.015372799833615622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,65536,0.10150187015533448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,7168,0.014051199952761332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,6144,0.012755200266838074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,4096,0.009975467125574749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,5120,0.011268267035484314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3584,0.009422933061917622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1024,0.005947733422120413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,3072,0.008341333270072937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2560,0.008077866832415263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,2048,0.007088000078996022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,1536,0.006934399902820587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,128,0.004770133395989736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,768,0.005554133156935374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,512,0.00525439977645874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,1024,256,0.00489279975493749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,16384,0.025895466407140095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,8192,0.015131733814875283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,65536,0.09554666678110758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,12288,0.020660267273585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,6144,0.012423466642697651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,10240,0.018125865856806436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,7168,0.013796266913414002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,5120,0.011053867141405741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3072,0.008319999774297078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,4096,0.00979306697845459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,3584,0.009402666489283245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2560,0.008062933385372163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,2048,0.0070602665344874065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1536,0.006839466591676076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,1024,0.005907199780146281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,512,0.0051701332132021586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,256,0.004889599978923798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,768,0.005465599894523621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,16384,0.025814400116602583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,12288,0.02020906607309977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,768,128,0.004759466648101807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,10240,0.017850667238235474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,65536,0.09060586293538411
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,8192,0.014631467064221701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,7168,0.013430399696032205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,5120,0.011073066790898641
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3584,0.009444266557693481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,4096,0.009741866588592529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,6144,0.012362666924794515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2560,0.008028799792130788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1536,0.006792533397674561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,3072,0.008268799881140392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,2048,0.007039999961853028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,256,0.004850133260091146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,512,0.005188266436258951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,1024,0.005967999994754791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,768,0.005448533097902933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,16384,0.025075199206670125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,12288,0.019733333587646486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,65536,0.0899061361948649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,10240,0.017192532618840538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,512,128,0.004715733230113983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,8192,0.014571733276049294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,7168,0.013431466619173684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,6144,0.012309333682060242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,5120,0.011100799838701884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,4096,0.009867733716964722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3584,0.009348266323407491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,3072,0.008313600222269695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2560,0.008027733365694682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1536,0.00678719977537791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,1024,0.005841066439946493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,768,0.005530666808287303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,256,0.004851200183232625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,512,0.005163733164469401
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,2048,0.0070826664566993715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,256,128,0.004675200084845225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,12288,0.019735467433929444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,10240,0.017207467555999757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,65536,0.0893397331237793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,8192,0.014565333724021912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,16384,0.024946133295694985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,5120,0.011170132954915365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,7168,0.01344319979349772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,6144,0.012293333808581035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,4096,0.009746133287747701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3584,0.009265066186587016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2048,0.007062399884064992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,3072,0.008227199812730153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,768,0.005399466554323832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,512,0.005143466591835022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,2560,0.00795413355032603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1536,0.006837333242098491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,1024,0.0059797331690788266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,128,0.004668800036112467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,128,128,256,0.004792533318201701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,7168,0.16709225972493488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,8192,0.18857812881469727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,10240,0.23440747261047362
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,12288,0.2761514663696289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,6144,0.14602239926656085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,4096,0.10221760272979737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,16384,0.360096009572347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,5120,0.12411306699117024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3584,0.09427093664805095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2048,0.06356053352355957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,3072,0.08056106567382812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1536,0.04973973433176677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,2560,0.0692255973815918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,256,0.01778986652692159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,128,0.01617280046145121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,512,0.02341653307278951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,768,0.030748800436655684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,65536,1024,0.03456533352533976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,8192,0.054117333889007566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,10240,0.06496106783548991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,12288,0.07679466406504312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,16384,0.09908373355865478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,7168,0.04852586587270101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,6144,0.04307200113932292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,5120,0.03720959822336833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,4096,0.03178986708323161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2560,0.022664533058802287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3072,0.02574399908383687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,3584,0.02872640093167623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,2048,0.019576533635457357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1024,0.012781866391499839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,1536,0.015787733594576518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,768,0.011494400103886922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,512,0.009821866949399311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,128,0.006866133213043213
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,256,0.007485866546630859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,12288,0.06100053389867147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,16384,0.07936426798502604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,10240,0.05197973251342773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,7168,0.03851413329442342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,8192,0.04316800038019816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,6144,0.03389546473821004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,16384,65536,0.36480639775594076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3072,0.02261120080947876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,4096,0.025127466519673663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,5120,0.029602134227752687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,3584,0.0229312002658844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2560,0.017910399039586387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,2048,0.015123200416564942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1536,0.012904533743858337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,1024,0.010575999816258747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,768,0.009454933802286784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,512,0.007392000158627827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,256,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,128,0.005962666869163513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,12288,65536,0.29212799072265627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,8192,0.041568001111348465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,12288,0.05426880121231079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,10240,0.046323200066884354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,16384,0.06981120109558106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,5120,0.025988266865412397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,6144,0.03031466603279114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,7168,0.03378560145696004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,4096,0.022266666094462075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2560,0.01562879979610443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,2048,0.013690666357676188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3072,0.01770346760749817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,3584,0.020284799734751384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1024,0.009547733267148336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,1536,0.011504000425338745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,768,0.008280533552169799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,512,0.006986666719118755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,256,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,128,0.005554133156935374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,16384,0.059638400872548424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,10240,0.04100906848907471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,10240,65536,0.2592096010843913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,12288,0.04686400095621745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,8192,0.03270933429400126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,5120,0.022705066204071044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,6144,0.027944533030192058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,7168,0.02924799919128418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,4096,0.019476266702016194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2560,0.014566399653752646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3584,0.017846399545669557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,3072,0.016082132856051125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,2048,0.012434132893880208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,65536,0.23959147135416664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1024,0.008875733613967896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,768,0.007769600053628286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,512,0.007159466544787089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,1536,0.010874666770299276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,128,0.005770666897296906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,8192,256,0.006292266647020976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,10240,0.04079893430074056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,12288,0.046741334597269694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,16384,0.060082133611043295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,8192,0.03316906690597534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,6144,0.02573440074920654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,7168,0.02935360074043274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3584,0.017035732666651406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,5120,0.022631466388702393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,4096,0.01861226757367452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1536,0.01030399998029073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,3072,0.015305599570274353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2048,0.01185706655184428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,2560,0.013874133427937826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,65536,0.22470506032307944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,1024,0.008472533027331034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,768,0.00713919997215271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,512,0.0065194666385650635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,256,0.005830400188763936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,7168,128,0.005446400245030721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,12288,0.04456746578216553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,10240,0.038338132699330646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,16384,0.057394134998321536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,8192,0.03128106594085693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,5120,0.021000534296035767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,6144,0.02442880074183146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,4096,0.017642666896184288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,7168,0.027808000644048054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3584,0.016184533635775246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,3072,0.014963199694951376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2560,0.012949333588282267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,65536,0.22146560351053873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,2048,0.011340799927711486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1536,0.009958400328954061
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,1024,0.007650133470694225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,768,0.006886399785677592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,512,0.006262399752934774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,256,0.00561599979797999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,6144,128,0.005179733534653981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,10240,0.03657813469568889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,12288,0.042224001884460446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,8192,0.02991146643956502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,16384,0.0544106682141622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,7168,0.02674773335456848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,5120,0.019908267259597778
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,4096,0.01691733400026957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,6144,0.02352213263511658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3584,0.015657599767049155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,65536,0.20455466906229652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,3072,0.014079999923706055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2560,0.012680533528327941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1536,0.009469866752624512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,2048,0.011109333237012227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,1024,0.00761706680059433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,768,0.006810666620731353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,512,0.006206933160622915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,256,0.005584000051021576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,5120,128,0.005167999863624572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,10240,0.0320032000541687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,8192,0.027866667509078978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,12288,0.03699306646982829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,16384,0.04937599897384644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,7168,0.02403200070063273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,6144,0.021336533625920615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,5120,0.017992534240086875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,4096,0.0160970667997996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3072,0.013421866297721862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,65536,0.18634880383809407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,3584,0.014684800306955972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2560,0.011793067057927448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,2048,0.010531199971834819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1536,0.008836266398429871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,1024,0.0071946665644645694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,768,0.0067210664351781205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,256,0.00555733342965444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,512,0.006146133442719777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,4096,128,0.005121066669623057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,10240,0.031015467643737794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,16384,0.04543360074361165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,12288,0.03572373390197754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,8192,0.02640320062637329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,7168,0.023524266481399537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,6144,0.020407466093699138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,5120,0.017991467316945394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,65536,0.16868693033854168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,4096,0.015210666259129844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3584,0.014250666896502177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,3072,0.012968533237775168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2560,0.01190826694170634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1536,0.008356266220410665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,2048,0.00999679962793986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,1024,0.006989866495132446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,768,0.00655680000782013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,512,0.006009600063165029
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,256,0.005479466418425242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3584,128,0.005191466708978017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,16384,0.048553601900736494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,12288,0.037213865915934244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,10240,0.03323413332303365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,8192,0.027118933200836182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,7168,0.02423786719640096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,65536,0.1869770685831706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,6144,0.02148373325665792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,5120,0.01883946657180786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,4096,0.015637333194414772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3584,0.014261333147684732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2560,0.011814399560292562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1024,0.0072543998559316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,3072,0.012982400258382163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,2048,0.010196266571680705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,1536,0.008436266581217449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,768,0.006538666784763336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,512,0.0059690664211908976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,128,0.005020800232887268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,3072,256,0.005357866485913595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,16384,0.04540479977925618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,10240,0.030930133660634358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,12288,0.034729599952697754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,8192,0.025552000602086383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,4096,0.01434346636136373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,65536,0.16560533841451008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,7168,0.02330453395843506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,5120,0.01715839902559916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,6144,0.02018453280131022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3584,0.01320319970448812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2560,0.010739200313886007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,3072,0.011756799618403117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,2048,0.009107200304667155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1536,0.007981866598129272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,768,0.00626986672480901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,1024,0.006899199883143107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,512,0.005790933469931285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,128,0.005025066435337067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2560,256,0.00527999997138977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,12288,0.03359466791152954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,10240,0.02882560094197591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,16384,0.04174826542536418
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,8192,0.024246400594711302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,65536,0.1669578711191813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,7168,0.021597866217295328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,5120,0.01660160024960836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,6144,0.01936960021654765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,4096,0.014522666732470194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2560,0.010380799571673077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,2048,0.009317333499590557
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3584,0.013405866424242654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,3072,0.011788800358772278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1536,0.008148266871770223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,1024,0.006885333359241486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,768,0.006312533219655354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,512,0.0057429333527882894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,256,0.00521066685517629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,2048,128,0.004973866542180379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,12288,0.03022186756134033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,16384,0.03890453179677327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,7168,0.02026453415552775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,10240,0.02581760088602702
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,8192,0.022436267137527464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,65536,0.13799254099527997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,6144,0.017127466201782227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,5120,0.015147733688354491
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,4096,0.01232426663239797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3584,0.011746133367220562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,3072,0.010376532872517902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2560,0.009734400113423665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1536,0.007339733342329661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,2048,0.008387200037638346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,1024,0.006343466540177663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,768,0.005924266576766968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,512,0.005515733361244201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,256,0.005132799843947092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1536,128,0.004863999783992767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,16384,0.031217066446940105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,10240,0.02204906741778056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,65536,0.11733120282491047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,12288,0.023119999965031942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,7168,0.016381866733233132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,5120,0.012334932883580525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,4096,0.010054399569829304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,8192,0.01775040030479431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,6144,0.014382933576901754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2560,0.008165333171685536
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3584,0.009742933511734008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,3072,0.008756267031033833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,2048,0.007371733089288075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1536,0.006932266553243001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,1024,0.005929600199063619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,256,0.004980266590913137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,768,0.005594666798909505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,512,0.005348266661167144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,1024,128,0.004810666541258494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,16384,0.027180800835291546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,10240,0.019684267044067384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,8192,0.016198399662971496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,7168,0.013801599542299906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,12288,0.022502400477727256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,5120,0.010790399710337321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,4096,0.00936853289604187
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,65536,0.09313813050587973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3584,0.00934933324654897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,6144,0.012216533223787945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2560,0.008066133161385854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,2048,0.007009066641330719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1536,0.0068693334857622785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,3072,0.008254933357238769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,512,0.005206400156021118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,768,0.00554666668176651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,256,0.004917333523432413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,1024,0.005948799848556519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,768,128,0.004775466521581014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,16384,0.024864000082015992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,10240,0.016795732577641807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,8192,0.014587733149528503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,12288,0.019642666975657145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,6144,0.011914666493733723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,7168,0.01330880026022593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,65536,0.08640213012695312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,4096,0.009291733304659527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3584,0.009221333265304565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,3072,0.008162133395671844
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,5120,0.010558933019638062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2560,0.007923200229803721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,2048,0.006980266670385997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1536,0.0067456002036730455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,1024,0.005776000022888183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,768,0.00555626650651296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,256,0.004839466512203216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,512,0.005111466844876607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,512,128,0.004717866579691568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,16384,0.02410773237546285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,65536,0.08586026827494303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,12288,0.01920959949493408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,10240,0.01660586694876353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,8192,0.014432000120480857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,5120,0.01055680016676585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,7168,0.01322773297627767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,6144,0.011879466970761617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,4096,0.00929813285668691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3072,0.008184533317883809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2048,0.006974933544794719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1536,0.006692266464233399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,3584,0.009175466497739156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,1024,0.005835733314355215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,2560,0.007892266909281413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,768,0.005340800185998281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,512,0.005110399921735128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,256,0.004852266609668731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,256,128,0.004631466666857402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,10240,0.01664959987004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,12288,0.019218132893244425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,16384,0.02400426665941874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,65536,0.0852672020594279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,7168,0.013221333424250284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,5120,0.010487467050552368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,8192,0.014475733041763306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,4096,0.009291733304659527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,6144,0.01190613309542338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2560,0.007831466694672901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1536,0.0067445332805315655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3584,0.009055999914805095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,3072,0.008089600006739299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,2048,0.0068810666600863145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,768,0.005354666709899902
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,256,0.004761599997679392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,1024,0.005864533285299936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,128,0.004609066744645437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,96,128,512,0.005076266825199127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,7168,0.16866453488667804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,8192,0.19138026237487793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,10240,0.24146347045898436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,12288,0.28229761123657227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,5120,0.12537279923756917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,4096,0.10055253505706788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,6144,0.14757653872172039
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3584,0.09295360247294107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,16384,0.3820373217264811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1024,0.036917332808176676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2048,0.057178668181101476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,1536,0.045661866664886475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,2560,0.06871253649393717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,3072,0.07989973227183024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,256,0.017296000321706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,128,0.015762133399645488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,768,0.030458666880925495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,65536,512,0.023005867004394533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,8192,0.05377813180287679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,10240,0.06635946830113729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,16384,0.10039680004119873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,12288,0.07731839815775553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,7168,0.048102398713429764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,5120,0.03688426812489827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,4096,0.03131306568781535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,6144,0.04291946490605672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3072,0.02540053327878316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,3584,0.02866986592610677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2560,0.022498132785161336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1536,0.01575573285420736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,2048,0.0192138671875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,1024,0.012640000383059184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,768,0.011267200112342834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,512,0.009693866968154908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,256,0.0075093333919843035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,128,0.006762666503588359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,10240,0.054764799276987707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,16384,0.08235946496327719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,12288,0.06441386540730795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,8192,0.04410880009333293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,7168,0.039082666238149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,6144,0.03431253433227539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,16384,65536,0.3679306666056315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,5120,0.029875200986862183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,4096,0.02508373260498047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2048,0.015331199765205384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3584,0.02298240065574646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,3072,0.02244480053583781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,2560,0.018051199118296304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1536,0.012739200393358865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,1024,0.010573866963386535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,768,0.00941973328590393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,128,0.005850666761398315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,512,0.007635200023651123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,256,0.00647680014371872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,12288,65536,0.29816532135009766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,10240,0.05020693143208822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,12288,0.058540801207224526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,16384,0.07894720236460367
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,5120,0.02747093240420024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,6144,0.03121066689491272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,8192,0.042251733938852946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,7168,0.036423468589782716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,4096,0.02317333420117696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3584,0.02121493419011434
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2048,0.014146133263905843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,3072,0.018617600202560425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,2560,0.016335999965667723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,768,0.008658132950464885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1536,0.011645866433779399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,1024,0.009614933530489604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,512,0.007018666466077168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,128,0.005469866593678792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,256,0.006157866617043813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,10240,65536,0.273466682434082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,12288,0.05066239833831787
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,10240,0.04329813321431478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,16384,0.06300053199132284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,8192,0.0354144016901652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,6144,0.02686400016148885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,5120,0.0237119992574056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,7168,0.030230400959650676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,4096,0.020087466637293497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3584,0.018965333700180054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,3072,0.016662399967511496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1024,0.009201066692670186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2560,0.01499626636505127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,2048,0.012986666957537332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,1536,0.010949333508809406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,65536,0.2528554598490397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,512,0.007245866457621257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,256,0.006455466647942861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,768,0.008467200398445129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,8192,128,0.005805866420269012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,10240,0.04503680070241292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,12288,0.05176533460617065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,16384,0.0643402655919393
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,8192,0.03547306855519612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,5120,0.023785599072774253
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,7168,0.03255040049552917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,6144,0.02740266720453898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3072,0.01704746683438619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2560,0.015006933609644571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,4096,0.02067199945449829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,3584,0.01858133276303609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,2048,0.01309333344300588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1536,0.010662399729092916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,65536,0.26454079945882164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,1024,0.008671999971071879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,512,0.006667733192443848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,768,0.007694933315118153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,256,0.005937066674232483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,7168,128,0.0053727999329566956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,10240,0.04287466605504354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,12288,0.04898666540781657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,8192,0.03519573211669922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,16384,0.0643338680267334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,5120,0.023625600337982177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,4096,0.01978773276011149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,6144,0.0260917325814565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,7168,0.030148265759150188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3072,0.016461867094039916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1536,0.010506666700045268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,3584,0.018196266889572144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2560,0.014367999633153281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,2048,0.012426666418711345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,65536,0.2381813367207845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,1024,0.007971199850241344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,768,0.007396266857783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,512,0.006393600006898243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,256,0.005756799876689911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,6144,128,0.005268266797065735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,12288,0.0470197319984436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,16384,0.06346346537272135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,10240,0.04139839808146159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,8192,0.03290560046831767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,6144,0.026733867327372235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,7168,0.029469867547353108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,4096,0.019003732999165853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,5120,0.022175999482472737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2560,0.0139765332142512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,65536,0.23840746879577637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3584,0.017669334014256795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,3072,0.015715199708938598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,2048,0.01188693344593048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1536,0.009760000308354696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,256,0.005718400080998739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,1024,0.00787306676308314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,768,0.007351466516653697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,512,0.00631573349237442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,5120,128,0.005182933310667674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,16384,0.052390400568644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,10240,0.035588268438975015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,12288,0.03937919934590657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,5120,0.019397334257761637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,8192,0.027903999884923297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,7168,0.025513599316279095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,6144,0.021834667523701987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,65536,0.20583359400431314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,4096,0.016492799917856852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3584,0.015277866522471109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,3072,0.01399893363316854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2560,0.012247467041015625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1536,0.009194667140642803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,2048,0.010963199536005656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,512,0.006321066617965698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,1024,0.007423999905586243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,768,0.007227733234564463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,256,0.005623466769854228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,4096,128,0.005248000224431356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,12288,0.03958400090535481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,8192,0.027668267488479614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,16384,0.050327467918396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,10240,0.035445332527160645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,5120,0.01872640053431193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,7168,0.02565760016441345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,4096,0.016495999693870545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,6144,0.021588265895843506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3584,0.015729066729545594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,65536,0.18587199846903485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2048,0.010318932930628459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,3072,0.01402773360411326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,2560,0.012476799885431926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,768,0.007142400244871776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1536,0.008638933300971985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,1024,0.007316266496976216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,512,0.006205866734186808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,128,0.00522986650466919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3584,256,0.005604266623655955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,16384,0.05555733442306519
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,12288,0.04302613337834676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,8192,0.031653332710266116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,10240,0.03853546778361003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,6144,0.024935466051101685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,7168,0.027245867252349853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,5120,0.021477333704630532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,4096,0.017940266927083334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3072,0.014779733618100485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,65536,0.21318079630533854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2048,0.010917333761850993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,3584,0.016617600123087564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1536,0.008796800176302593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,512,0.006122666597366333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,2560,0.013268267114957174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,1024,0.007362133264541626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,768,0.0070250665148099255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,256,0.005342933535575867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,16384,0.05166186491648356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,12288,0.040533332029978435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,3072,128,0.0050122668345769245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,10240,0.03653013308842977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,8192,0.02866133252779643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,7168,0.025679999589920045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,5120,0.019732266664505005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,4096,0.016525866587956746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,6144,0.023099732398986817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,65536,0.18825920422871906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3072,0.01322879989941915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,3584,0.015004799763361613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1024,0.0072543998559316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2560,0.011589333415031433
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,1536,0.008553600311279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,2048,0.010134399930636088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,256,0.005387733379999796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,128,0.004983466863632202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,768,0.006695466736952464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2560,512,0.005997866888840993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,16384,0.04612373510996501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,8192,0.025591466824213666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,12288,0.03727999925613403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,10240,0.03262613415718078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,65536,0.189572270711263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,5120,0.01862186590830485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,7168,0.02341973384221395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3584,0.014267733693122864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,6144,0.020107734203338622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2560,0.010931199789047242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,4096,0.015370666980743408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1536,0.008280533552169799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,1024,0.007152000069618225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,3072,0.01263146698474884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,2048,0.009725866715113322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,256,0.005474133292833964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,512,0.006021333237489065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,768,0.00687360018491745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,2048,128,0.004948266843954722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,10240,0.029989333947499593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,16384,0.043057068188985186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,8192,0.02491626739501953
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,12288,0.03406399885813395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,65536,0.16203734079996746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,7168,0.02204586664835612
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,6144,0.019450666507085164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,5120,0.016936532656351724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,4096,0.013822933038075766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2560,0.010331733028093974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3584,0.013037866353988648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,3072,0.011521066228548687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,768,0.006409599880377452
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,2048,0.009106133381525676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1536,0.007749333480993907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,1024,0.006681600213050842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,128,0.004859733581542969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,512,0.005679999788602194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1536,256,0.00513919989267985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,16384,0.035426131884257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,12288,0.027914667129516603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,10240,0.02566186587015788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,8192,0.02086826761563619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,65536,0.12914559841156006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,7168,0.018198400735855103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,6144,0.016084266702334087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,5120,0.013605333367983499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3584,0.0107722669839859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,3072,0.009939199686050415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2560,0.008992000420888265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,2048,0.007944533228874206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,4096,0.011509333054224651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1536,0.007053866485754649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,768,0.005751466751098633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,512,0.005366399884223938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,256,0.0049685334165891016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,1024,0.006151466568311056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,10240,0.021744000911712646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,1024,128,0.004734933376312256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,65536,0.10325547059377034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,12288,0.023496532440185548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,16384,0.031016532580057782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,8192,0.01643946667512258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,7168,0.014734933773676554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,6144,0.012231466174125672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3584,0.00944533348083496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,4096,0.009872000416119893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2560,0.008020266890525818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,5120,0.011688533425331115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,3072,0.008671999971071879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,768,0.005659733215967814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,2048,0.007197866837183635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,512,0.005310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1536,0.006821333368619282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,1024,0.005871999760468801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,256,0.004920533299446106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,768,128,0.004698666433493296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,16384,0.025751467545827227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,10240,0.016924800475438435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,12288,0.019349332650502524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,8192,0.014358400305112203
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,65536,0.09371519883473714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,7168,0.013077333569526672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,6144,0.011601066589355469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,5120,0.01050453285376231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,4096,0.009236266215642292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3584,0.009191466371218364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,3072,0.008098133405049642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1536,0.006679466863473256
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,1024,0.005824000140031179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2560,0.007853866616884867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,2048,0.00691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,768,0.005512533088525137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,512,0.005221333106358846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,256,0.0048991998036702475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,512,128,0.004669866462548574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,65536,0.08371307055155436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,12288,0.01877440015474955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,16384,0.023784534136454264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,8192,0.014192000031471252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,7168,0.01304639975229899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,6144,0.011520000298817952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,5120,0.01048959990342458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,10240,0.016567466656366985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,4096,0.009271466732025146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3072,0.008116266628106435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2560,0.007820799946784973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,3584,0.009115733702977498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,2048,0.006974933544794719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1536,0.006690133114655812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,1024,0.005771733323733012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,512,0.005128533144791921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,256,0.004799999793370565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,128,0.0045952002207438145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,256,768,0.005456000069777171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,16384,0.023690666755040488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,65536,0.08336533705393473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,12288,0.01880426605542501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,10240,0.016566399733225504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,7168,0.013064533472061157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,8192,0.014269866545995078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,6144,0.011577600240707397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3584,0.008973866701126099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,3072,0.008028799792130788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2560,0.007786666850248973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,5120,0.010469333330790202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,4096,0.009172266721725464
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1024,0.00581333339214325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,768,0.005351466437180838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,1536,0.006694399813810985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,512,0.005016533533732096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,2048,0.006877866884072621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,256,0.004789333542188009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,80,128,128,0.004555733501911163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,7168,0.16315840085347494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,8192,0.18478186925252277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,10240,0.231331205368042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,12288,0.27226241429646814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,16384,0.35588480631510416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,6144,0.14242773056030272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,5120,0.12040106455485027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,4096,0.09939946333567301
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3584,0.09064640204111735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,3072,0.08573119640350342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2048,0.054495998223622645
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,2560,0.06556373437245687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1024,0.03229653239250183
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,1536,0.043329068024953205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,512,0.020862932999928793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,768,0.02632213234901428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,256,0.014513066411018372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,65536,128,0.012342400352160136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,8192,0.052084267139434814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,10240,0.06328426599502564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,12288,0.07458346684773763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,16384,0.09589440027872721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,5120,0.03550613323847453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,7168,0.046514133612314865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,6144,0.04102933406829834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,4096,0.029846400022506714
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3072,0.024141865968704223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2048,0.01804800033569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,2560,0.021409066518147786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,3584,0.02722773353258769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,65536,0.3616437276204427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,512,0.008674133817354839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,768,0.010124799609184266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1024,0.011583999792734782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,256,0.006260266900062561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,1536,0.014591999848683677
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,16384,128,0.005966933568318685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,12288,0.05803093512852987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,16384,0.07388266722361246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,10240,0.04918826818466186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,8192,0.04115946690241496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,7168,0.03656853437423706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,6144,0.03231040040651957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,5120,0.028086400032043456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,4096,0.023896533250808715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3072,0.019266132513682047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,3584,0.021660800774892172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,65536,0.2741045316060384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2560,0.016812799374262492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,2048,0.014403200149536133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1536,0.011878400047620138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,768,0.009453866879145305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,512,0.006444799900054932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,256,0.005585066477457682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,1024,0.009901866316795349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,12288,128,0.00517546683549881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,10240,0.04240533510843913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,12288,0.04968213240305583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,16384,0.06357760032018026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,8192,0.03563733498255412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,6144,0.027896533409754436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,7168,0.0315008004506429
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,4096,0.020702934265136717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,5120,0.024322134256362916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3072,0.016778665781021117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,65536,0.2325493335723877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,3584,0.018645334243774413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2560,0.014629333217938741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,2048,0.012417067090670269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1536,0.0107914666334788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,768,0.007613866527875264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,512,0.005717333157857259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,1024,0.009769599636395771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,256,0.0052255998055140175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,10240,128,0.004994133114814758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,10240,0.03535573482513428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,12288,0.041857067743937174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,16384,0.052323198318481444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,8192,0.029793065786361695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,7168,0.02648426691691081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,6144,0.023612799247105916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,5120,0.020683733622233073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,4096,0.01745706597963969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,65536,0.18910293579101561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3072,0.014069333672523499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2560,0.012988799810409546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,3584,0.01614293356736501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,2048,0.011362133423487346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1536,0.010797866185506185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,1024,0.008091733356316884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,256,0.005382399757703145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,768,0.006502399841944377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,512,0.005735466877619425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,8192,128,0.005151999990145365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,12288,0.0378602663675944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,10240,0.03246080080668132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,16384,0.04810773531595866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,8192,0.0271178662776947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,6144,0.021651200453440347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,7168,0.024307199319203696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,5120,0.018953599532445273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,4096,0.01590506633122762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,65536,0.17115306854248047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3072,0.01295360028743744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,3584,0.014487466216087342
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2560,0.011928533514340717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,2048,0.010375466942787171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1536,0.00985599954922994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,1024,0.0070933332045873005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,768,0.005770666897296906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,512,0.005464533468087515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,256,0.005082666873931885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,7168,128,0.004837333162625631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,12288,0.03372906843821208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,16384,0.042513068517049155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,10240,0.029033599297205608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,8192,0.024171733856201173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,7168,0.021699200073877968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,6144,0.019384533166885376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,4096,0.014349866906801859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,5120,0.01677653392155965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,65536,0.15126506487528485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3584,0.01320746640364329
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,3072,0.011958400408426922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2560,0.010905599594116211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1536,0.009264000256856282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,2048,0.009630933403968811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,768,0.005486933390299479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,1024,0.006102400024731954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,512,0.005257600049177805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,256,0.004971733192602793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,6144,128,0.004743466774622599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,12288,0.030210133393605548
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,8192,0.021871999899546305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,10240,0.026151466369628906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,16384,0.03833173513412476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,6144,0.017436800400416057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,5120,0.015176533659299215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,7168,0.019758933782577516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,4096,0.013159466783205667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,65536,0.13332479794820148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3584,0.012281599640846252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,3072,0.011055999994277954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2560,0.010228266318639118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,2048,0.009195733070373534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1536,0.008036266764005024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,1024,0.005952000121275584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,768,0.005534933507442474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,512,0.005222400029500326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,256,0.004962133367856344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,5120,128,0.004801066716512045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,8192,0.018294399976730345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,12288,0.025425066550572712
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,10240,0.021699200073877968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,16384,0.031674667199452715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,5120,0.01304746667544047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,7168,0.016356266538302102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,6144,0.014754133423169455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,4096,0.011610666910807293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,65536,0.10893332958221436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2048,0.008140799899895985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3072,0.009956266482671101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,2560,0.009409067034721375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,3584,0.010982400178909302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1536,0.006769066552321117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,1024,0.005639466643333435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,256,0.004870399832725525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,512,0.005239466826121012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,768,0.005425066749254862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,4096,128,0.004732800026734671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,8192,0.017087999979654947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,12288,0.023605332771937052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,10240,0.02037013371785482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,16384,0.02932479977607727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,6144,0.013901866475741067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,7168,0.015130666891733804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,65536,0.10197227001190186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,5120,0.01241919994354248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,4096,0.011012267072995503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2048,0.007539199789365132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3072,0.009591466188430786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,3584,0.010589866836865743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,2560,0.009798399607340495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1536,0.006533333162466686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,256,0.004930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,512,0.005234133203824361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,128,0.004698666433493296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,1024,0.00558186670144399
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3584,768,0.0054282665252685545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,10240,0.02029866576194763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,8192,0.0170197327931722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,12288,0.023534933725992836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,16384,0.029513599475224812
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,65536,0.10472640196482341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,6144,0.013736533125241599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,7168,0.015209600329399109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,4096,0.010942932963371278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,5120,0.012427733341852824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3584,0.010319999853769938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,3072,0.009381332993507385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2560,0.008422399560610454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1536,0.006190933287143707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,2048,0.007063466807206471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,512,0.0050346667567888895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,256,0.004819199939568837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,1024,0.005673600236574808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,768,0.005258666475613912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,3072,128,0.0046623999873797095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,12288,0.02228053410847982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,10240,0.01937920053799947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,8192,0.016295466820398966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,16384,0.027850667635599773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,65536,0.09886826674143473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,7168,0.014734933773676554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,6144,0.013252266248067222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,4096,0.010651733477910359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,5120,0.012025599678357441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3584,0.010046933094660442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,3072,0.008845866719881693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2560,0.008088533580303193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1536,0.006166400015354156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,2048,0.007125333448251088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,768,0.005329066514968872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,1024,0.005623466769854228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,512,0.005047466854254405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,128,0.004699733356634776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2560,256,0.004766933123270671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,16384,0.02318293253580729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,10240,0.01638826628526052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,8192,0.014174933234850565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,12288,0.018636800845464072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,65536,0.08215040365854899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,7168,0.012776533762613932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,5120,0.010729599992434185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,6144,0.011799466609954835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,4096,0.009525332848230999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3584,0.009066667159398396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,3072,0.007797333101431529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2560,0.0074538667996724445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,2048,0.0065610667069753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1536,0.006152533491452535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,1024,0.005458133419354757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,768,0.005161599814891815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,128,0.004678399860858917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,256,0.00477866679430008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,2048,512,0.005008000135421753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,16384,0.02220053275426229
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,12288,0.018130133549372353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,10240,0.016085333625475564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,8192,0.013819733262062072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,65536,0.07855253219604492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,7168,0.012588799993197123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,6144,0.011451733112335206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,5120,0.010249599814414978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,4096,0.008778666456540424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3584,0.00863039990266164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2048,0.006493866443634033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,3072,0.0075541332364082335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,2560,0.00722453345855077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1024,0.005376000205675761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,1536,0.0061141331990559895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,256,0.004749866823355356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,128,0.004582400123278299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,512,0.004936533172925314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1536,768,0.005110399921735128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,65536,0.07622719605763753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,16384,0.02182719906171163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,12288,0.017991467316945394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,10240,0.015877333283424378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,8192,0.013727999726931252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,7168,0.012152533729871113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,6144,0.010899200042088827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,5120,0.009604266285896302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,4096,0.008503466844558716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3584,0.008301866551240284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,3072,0.007422933479150136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2560,0.007132799923419952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,2048,0.006401066482067108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1024,0.005413333574930826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,1536,0.006064000229040781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,768,0.005093333125114441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,256,0.004666666686534882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,512,0.004855466882387797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,1024,128,0.004563199977080027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,65536,0.07555733521779379
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,16384,0.021911466121673585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,12288,0.017876267433166504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,10240,0.015571199854214988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,8192,0.013169067104657492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,7168,0.011854933698972066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,6144,0.010614400108655293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,5120,0.009501866499582927
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,4096,0.008492799599965413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3584,0.00830506682395935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,3072,0.007479466497898102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2048,0.0064746667941411335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,2560,0.007169066866238911
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1536,0.006093866626421611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,1024,0.005412266651789347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,768,0.005066666503747304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,512,0.004846933484077454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,256,0.004645333190759023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,768,128,0.004485333462556204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,65536,0.07517013549804688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,16384,0.02181866765022278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,12288,0.017429333925247193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,10240,0.015085867047309876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,8192,0.012793599565823873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,7168,0.011565867066383361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,6144,0.010461866855621338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,5120,0.009483733773231506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,4096,0.008455466230710347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3584,0.008241066833337148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,3072,0.007447466750939687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2560,0.007117866476376851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,2048,0.006410666803518932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1024,0.005314133564631144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,1536,0.006040533383687338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,768,0.005041066805521647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,512,0.004766933123270671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,256,0.0046304002404212955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,512,128,0.0044821331898371375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,65536,0.07447360356648763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,16384,0.020920532941818237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,12288,0.01692053278287252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,10240,0.014904533823331198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,8192,0.012813867131868998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,7168,0.011508267124493916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,6144,0.01050986647605896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,5120,0.009452799956003826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,4096,0.00848746697107951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3584,0.008249600231647492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,3072,0.007410133381684621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2560,0.007089066505432129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,2048,0.006353066861629486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1536,0.006087466577688853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,1024,0.005287466446558634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,512,0.004816000163555145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,768,0.005028266708056132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,256,0.004523733258247375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,256,128,0.0044725333650906885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,65536,0.07441706657409668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,16384,0.020890667041142782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,10240,0.014817066987355552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,12288,0.01692906618118286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,8192,0.012829867005348206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,7168,0.011485866705576579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,6144,0.010455466310183207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,5120,0.009463466207186381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,4096,0.00844053328037262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3584,0.00823040008544922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,3072,0.007416533430417378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2048,0.0064064001043637585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,2560,0.007155199845631917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1024,0.005299200117588043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,1536,0.006000000238418579
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,768,0.005065600077311197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,512,0.004771199822425842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,256,0.004582400123278299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,64,128,128,0.004445866743723551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,7168,0.16196373303731282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,8192,0.18477226893107096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,10240,0.2272480010986328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,12288,0.27038825352986656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,16384,0.35486186345418297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,6144,0.14115947087605793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,5120,0.11972800095876057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,4096,0.09778239727020263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3584,0.0900544007619222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,3072,0.08519786993662516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1536,0.04241173267364502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,1024,0.031515733400980635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2048,0.05420480171839396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,2560,0.06471893390019735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,256,0.014327466487884521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,128,0.012226133545239767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,512,0.020163200298945107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,65536,768,0.025579732656478883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,8192,0.05137386719385782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,10240,0.06250666777292888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,12288,0.07408533096313477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,16384,0.09499413172403971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,6144,0.04072106679280599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,7168,0.04611839850743611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,5120,0.03510293165842692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,4096,0.029476267099380494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3072,0.02386773427327474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,3584,0.02706986665725708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2560,0.02114560008049011
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,2048,0.017964800198872886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,512,0.008708266417185466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1536,0.014808533589045205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,1024,0.011497599879900615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,768,0.010228266318639118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,256,0.006285866598288219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,128,0.005921066800753275
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,16384,65536,0.35448853174845374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,12288,0.057099731763203945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,16384,0.07378559907277425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,10240,0.04866986672083537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,8192,0.04585493405659993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,6144,0.03195199966430664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,7168,0.036134401957194015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,5120,0.02778880000114441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,4096,0.02359786629676819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3584,0.02148373325665792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2560,0.016664533813794454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,3072,0.018981333573659262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,2048,0.014158933361371358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1536,0.011772800485293071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,1024,0.009719467163085938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,768,0.008633599678675333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,512,0.006302933394908905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,65536,0.27139625549316404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,256,0.005564799904823304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,12288,128,0.005268266797065735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,10240,0.04174720048904419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,12288,0.05605866511662801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,8192,0.0349397341410319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,16384,0.06287999947865805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,7168,0.031217066446940105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,4096,0.020499199628829956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,6144,0.027691733837127686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,5120,0.024010666211446128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3584,0.01840426723162333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,3072,0.016383999586105348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1536,0.010627200206120808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2560,0.014547200004259745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,2048,0.012430933117866517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,1024,0.00894719958305359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,768,0.007441066702206929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,512,0.005906133353710175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,256,0.005348266661167144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,65536,0.22978347142537436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,10240,128,0.005053866902987162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,12288,0.04067306518554688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,16384,0.05177066723505656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,10240,0.039206401507059736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,8192,0.029483733574549358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,7168,0.026254934072494508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,4096,0.017462400595347087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,5120,0.020501333475112914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,6144,0.023382399479548135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3584,0.015998933712641397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,3072,0.014028799533843995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2560,0.01295360028743744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,2048,0.011174399654070537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,512,0.0059125334024429325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,65536,0.18767679532368978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1536,0.009955199559529622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,1024,0.008040533463160197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,768,0.0065749332308769224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,256,0.0054400001962979635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,8192,128,0.005293866495291392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,10240,0.0319701333840688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,12288,0.03734506766001384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,16384,0.04760746558507283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,8192,0.026678399244944258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,6144,0.021473066012064616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,7168,0.024094933271408082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,5120,0.018632533152898152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,4096,0.015780267119407655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,65536,0.168448003133138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3584,0.01437440017859141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2560,0.011813333630561829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,3072,0.012811733285586038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,2048,0.010478933652242024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1536,0.009782399733861287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,128,0.004881066580613455
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,1024,0.007006933291753133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,768,0.006296533346176148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,512,0.0056202664971351625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,7168,256,0.005109333495299021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,8192,0.02395306626955668
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,12288,0.03333333333333333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,10240,0.028602667649586994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,16384,0.04211093187332153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,7168,0.02164906660715739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,5120,0.01671573321024577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,4096,0.014173866311709086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,6144,0.019401599963506065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3584,0.01306880017121633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,3072,0.011900800466537475
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,65536,0.14875839551289877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2560,0.010940800110499065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,2048,0.009662933150927226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1536,0.008313600222269695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,1024,0.0064650664726893115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,256,0.0049898669123649595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,768,0.006040533383687338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,512,0.005444266895453135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,6144,128,0.004860800007979075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,12288,0.029736532767613726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,16384,0.037921067078908285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,10240,0.02576213280359904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,8192,0.021432532866795858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,7168,0.01949866612752279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,4096,0.013000532984733582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,6144,0.01744426687558492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,5120,0.015209600329399109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,65536,0.1330677350362142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3584,0.01206719974676768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1536,0.008211199939250947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,3072,0.01104639967282613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2560,0.010193066795667012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,2048,0.009086933732032777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,1024,0.006321066617965698
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,768,0.005961599946022034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,512,0.005407999952634176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,256,0.004977066814899444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,5120,128,0.0047775998711586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,12288,0.025183999538421632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,10240,0.02159573237101237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,16384,0.031804800033569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,8192,0.018196266889572144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,7168,0.016377600034077962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,6144,0.014949333667755128
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,4096,0.011550933122634888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,5120,0.01328426698843638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3584,0.011145599683125814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,65536,0.11368319988250733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,3072,0.010001066327095031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2560,0.009459199508031209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,2048,0.008146133522192638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1536,0.007096533477306366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,1024,0.006150400141874949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,768,0.005986133217811584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,512,0.005342933535575867
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,256,0.0050570666790008545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,4096,128,0.00486826648314794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,10240,0.020127999782562255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,12288,0.023171200354894003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,8192,0.01683626572291056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,16384,0.02939093311627706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,7168,0.015185067057609558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,65536,0.1024234692255656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,6144,0.01399679978688558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,5120,0.01262399951616923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,4096,0.010993066430091857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3584,0.010461866855621338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,3072,0.009614933530489604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2560,0.009064533313115438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,2048,0.007349333167076111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1536,0.006720000008742015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,1024,0.0058890665570894875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,768,0.005830400188763936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,256,0.004948266843954722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,512,0.005295999844868978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3584,128,0.004845866560935974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,16384,0.030108799537022907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,12288,0.023685334126154582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,8192,0.01695893406867981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,10240,0.02037973403930664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,65536,0.10864533583323162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,7168,0.015471999843915304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,6144,0.01421440045038859
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,5120,0.012537599603335062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,4096,0.011005866527557372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3072,0.009430399537086487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2560,0.008629332979520161
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,3584,0.01037440001964569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,2048,0.007291733225186665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1536,0.006549333532651265
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,512,0.005205333232879639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,768,0.005758933226267497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,1024,0.005961599946022034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,256,0.004890666902065277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,3072,128,0.004710400104522705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,16384,0.028305067618687944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,12288,0.021758933862050377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,10240,0.01895466645558675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,65536,0.09965546925862631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,8192,0.01616106629371643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,7168,0.01486186683177948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,6144,0.013359999656677246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,5120,0.012057600418726604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,4096,0.010663466652234395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3584,0.010001066327095031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,3072,0.008929066856702169
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2560,0.007805866499741872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,2048,0.006964266796906789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1536,0.006320000191529592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,1024,0.0059008002281188965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,768,0.00553706685702006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,512,0.005085866649945577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,256,0.0047978664437929785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2560,128,0.00472320020198822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,16384,0.025621332724889118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,12288,0.01990506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,10240,0.01696853240331014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,8192,0.014597333470980325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,65536,0.0914741357167562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,7168,0.013573333621025085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,6144,0.012442666292190551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,5120,0.011175466577212016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,4096,0.009809066851933796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3584,0.009166933099428813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2560,0.007582933207352956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,3072,0.008170666793982189
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,2048,0.006899199883143107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1536,0.006388266881306966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,1024,0.005749333401521047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,768,0.0056981335083643595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,512,0.00514026681582133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,256,0.004807466765244802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,2048,128,0.004726399978001913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,16384,0.02245866656303406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,12288,0.01795519987742106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,8192,0.013573333621025085
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,65536,0.07977920373280843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,10240,0.0156768004099528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,7168,0.012526933352152506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3584,0.008552533388137818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,6144,0.011487999558448791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,5120,0.010373333096504211
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,4096,0.008665600419044494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2048,0.0065290664633115125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,3072,0.0076117331782976794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,2560,0.007186133166154225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,768,0.00535999983549118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1024,0.0054954667886098225
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,1536,0.006017066538333893
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,512,0.005041066805521647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,256,0.004750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1536,128,0.004654933512210846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,16384,0.021724800268809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,8192,0.01323306659857432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,10240,0.01527253290017446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,65536,0.07448853651682535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,12288,0.01759679913520813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,5120,0.00970240036646525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,7168,0.011927466591199238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,6144,0.01074773371219635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,4096,0.008384000261624653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3584,0.008232533435026805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,3072,0.007423999905586243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2048,0.0064074665307998655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,2560,0.007053866485754649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1536,0.006026666859785716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,1024,0.005387733379999796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,768,0.005189333359400431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,512,0.004839466512203216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,256,0.004680533210436503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,1024,128,0.0045503998796145115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,16384,0.021865600347518922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,12288,0.017553067207336424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,65536,0.07357227007548015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,10240,0.015110400319099427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,8192,0.012921599547068277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,7168,0.011531733473141988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,6144,0.010363733768463135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,5120,0.009422933061917622
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,4096,0.00837546686331431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3584,0.008217599987983704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,3072,0.007418666779994964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2560,0.006968533496061961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,2048,0.006316799918810527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1536,0.005924266576766968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,1024,0.005365333457787832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,768,0.005065600077311197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,256,0.00461760014295578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,512,0.004829866687456766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,768,128,0.004488533238569895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,16384,0.0218122661113739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,65536,0.07311039765675863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,12288,0.016769067446390788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,10240,0.014599466323852539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,8192,0.012518399953842163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,7168,0.011377066373825073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,6144,0.010346666971842448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,5120,0.009432533383369445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,4096,0.008370133241017659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3584,0.008184533317883809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,3072,0.007389866809050243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2560,0.007032533486684163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,2048,0.00629013329744339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1536,0.005888000130653381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,1024,0.005268266797065735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,768,0.005082666873931885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,512,0.004801066716512045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,256,0.004603733122348785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,512,128,0.004479999840259552
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,65536,0.07212800184885661
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,16384,0.020806399981180827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,12288,0.016548267006874083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,10240,0.014300800363222756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,8192,0.012335999806722005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,7168,0.011377066373825073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,6144,0.010351999600728353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,5120,0.009425066908200582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3584,0.008148266871770223
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,4096,0.008339200417200725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,3072,0.007351466516653697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2560,0.006967466572920482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,2048,0.0063296000162760425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1536,0.005961599946022034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,1024,0.005236266553401947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,768,0.004957866668701172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,512,0.004782933493455251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,256,0.004557866851488749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,256,128,0.004459733267625173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,65536,0.0725279966990153
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,16384,0.020778665939966835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,10240,0.014282666643460593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,12288,0.016521599888801575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,8192,0.01234346628189087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,7168,0.011380267143249512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,6144,0.01034346620241801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,5120,0.009411199887593587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,4096,0.008323200047016144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3584,0.008221866687138875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,3072,0.007285333176453908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2560,0.007018666466077168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1536,0.005939200023810069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,2048,0.006278400123119354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,1024,0.005238399902979533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,512,0.004808533191680908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,768,0.0050346667567888895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,256,0.004567466676235199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,48,128,128,0.004454400142033895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,8192,0.1819104035695394
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,7168,0.1612181345621745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,10240,0.22577279408772788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,12288,0.2689845403035482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,6144,0.13932266235351562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,5120,0.11787947018941243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3584,0.08903466860453288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,4096,0.09679040114084879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,16384,0.358519458770752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1024,0.033923200766245526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2048,0.05308800141016642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,1536,0.04165120124816894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,2560,0.06383359829584757
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,3072,0.07527893384297689
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,256,0.013927466670672097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,512,0.01946773330370585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,768,0.02504533330599467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,65536,128,0.011872000495592753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,12288,0.0731935977935791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,8192,0.051015468438466396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,10240,0.06211413145065307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,16384,0.09404693444569906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,7168,0.045449598630269365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,5120,0.034714667002360027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,4096,0.029214932521184282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,6144,0.040139734745025635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3584,0.026565333207448322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2048,0.017451733350753784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1536,0.01444586714108785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,2560,0.020898133516311646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,3072,0.023543467124303184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,256,0.006485333542029063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,1024,0.011413333813349406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,768,0.010059733192125957
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,512,0.008416000008583068
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,128,0.0057888001203536986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,12288,0.0567360003789266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,16384,65536,0.3528287887573242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,10240,0.04813973506291707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,16384,0.0726698637008667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,5120,0.02741439938545227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,6144,0.0315829336643219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,8192,0.0401311993598938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,7168,0.04051733414332072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3584,0.021011199553807577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,4096,0.023270400365193684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2560,0.01655466655890147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,3072,0.018707199891408285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,768,0.008574933807055155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,2048,0.014133333166440328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1536,0.011638399958610535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,1024,0.009666132926940917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,256,0.005694933235645294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,512,0.006539733211199443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,128,0.005156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,12288,65536,0.26765225728352865
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,16384,0.0626250664393107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,12288,0.04866773287455241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,10240,0.041383465131123856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,8192,0.03907306591669719
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,7168,0.03090133269627889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,5120,0.02376746733983358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,6144,0.027324799696604413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,4096,0.02030186653137207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3584,0.01820053259531657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,3072,0.01627413332462311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1536,0.01058666706085205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2560,0.01440000037352244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,2048,0.012362666924794515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,768,0.007409066458543142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,1024,0.008851200342178345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,128,0.004986666639645894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,256,0.005490133166313171
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,512,0.006188799937566122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,10240,65536,0.2292341391245524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,8192,0.03252480030059814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,10240,0.03522773186365764
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,12288,0.04100906848907471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,16384,0.054478931427001956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,7168,0.026243199904759724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,5120,0.020454400777816774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,6144,0.023490132888158162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,4096,0.01776319940884908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2048,0.011332266529401143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3584,0.016029866536458333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,2560,0.01300373375415802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,3072,0.014154666662216186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1024,0.007987200220425924
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,1536,0.00983893374602
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,65536,0.1919861316680908
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,128,0.005171200136343638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,768,0.006799999872843425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,512,0.00618453323841095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,8192,256,0.0056415999929110205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,16384,0.05235626697540283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,10240,0.03461333513259888
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,12288,0.03957759936650594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,8192,0.029256532589594524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,6144,0.022396800915400188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,7168,0.025576533873875935
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,4096,0.016473600268363954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3584,0.01527466674645742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,5120,0.01954560081164042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2048,0.0105813334385554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,2560,0.012401066223780314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1536,0.009303466478983561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,1024,0.00761599987745285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,768,0.006433066725730896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,3072,0.013541332880655923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,256,0.005320533116658529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,65536,0.18348479270935059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,512,0.005859200159708659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,7168,128,0.004781866570313772
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,16384,0.04832746585210164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,12288,0.038466131687164305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,10240,0.032613333066304526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,8192,0.02759466568628947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,6144,0.02113706668217977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,7168,0.0240831991036733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,5120,0.018403200308481853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,4096,0.01562773287296295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2048,0.010168533523877461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,2560,0.011655466755231221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3584,0.014378666877746582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,3072,0.012990933656692506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,65536,0.1796138604482015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1536,0.008826667070388794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,768,0.006265600025653839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,512,0.005688533186912537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,1024,0.006886399785677592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,256,0.005190399785836538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,6144,128,0.00481386681397756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,12288,0.03514346679051717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,16384,0.04604693253835042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,8192,0.02645333409309387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,10240,0.03066026568412781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,6144,0.02039466698964437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,7168,0.022817067305246987
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,4096,0.015109333395957946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,5120,0.01735360026359558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3584,0.013920000195503235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,65536,0.16713493665059406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,3072,0.012378666798273723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2560,0.011358933647473653
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,2048,0.0099263995885849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1536,0.008444799979527792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,1024,0.00680213322242101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,512,0.005734399954477946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,256,0.005148800214131674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,128,0.004659200211366018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,5120,768,0.0060597335298856105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,12288,0.0320032000541687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,10240,0.027853866418202717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,8192,0.023382399479548135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,16384,0.04310400088628133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,6144,0.018488534291585288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,7168,0.020525866746902467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,5120,0.01612053314844767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,4096,0.014138666788736978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3072,0.011621333161989848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,65536,0.1432960033416748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2560,0.010435199737548828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,3584,0.012934399644533792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,2048,0.00939626693725586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,512,0.005624533196290334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1536,0.007773866752783458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,1024,0.006534400085608165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,768,0.006062933305899302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,256,0.005133866767088572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,4096,128,0.004773333172003428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,8192,0.021865600347518922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,12288,0.02995520035425822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,7168,0.019499733050664266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,16384,0.03909120162328084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,10240,0.0259552001953125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,6144,0.01745706597963969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3072,0.01125759979089101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,5120,0.015382400155067444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,65536,0.13248106638590496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,3584,0.0123690664768219
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,4096,0.013289599617322286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1536,0.007433600227038066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2560,0.010248532891273499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,2048,0.008714666962623597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,1024,0.006241066753864289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,768,0.0058378666639328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,512,0.005599999924500784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,128,0.004717866579691568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3584,256,0.0050687998533248905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,10240,0.0271232008934021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,12288,0.03136000037193298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,16384,0.041265066464742026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,8192,0.023118933041890465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,4096,0.01350826621055603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,7168,0.020553600788116456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,6144,0.018521600961685182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,5120,0.016017066438992818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,65536,0.1548949400583903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3072,0.011345066626866658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1536,0.007468800246715546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,3584,0.012602667013804117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2560,0.010014933347702027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,768,0.005809066692988077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,2048,0.008507733543713886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,1024,0.006351999938488007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,256,0.004986666639645894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,512,0.005497600138187409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,12288,0.028014934062957762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,3072,128,0.004698666433493296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,16384,0.03657706578572591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,7168,0.018411733706792197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,10240,0.024460800488789878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,65536,0.1395232041676839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,8192,0.021280000607172646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,6144,0.016536532839139303
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3584,0.011877333124478657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,3072,0.010518399874369304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2560,0.00902506709098816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,5120,0.014516266187032065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,4096,0.012409599622090657
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1024,0.006179200112819671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,512,0.005253333350022634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,768,0.005658666789531708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,2048,0.007739733159542084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,1536,0.00691840002934138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,256,0.004964266717433929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2560,128,0.004509866734345754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,16384,0.03709973494211833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,12288,0.02840320070584615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,8192,0.021186133225758873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,7168,0.01911466717720032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,10240,0.024819199244181314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,5120,0.014702933033307395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,65536,0.1345919926961263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,4096,0.012472533186276754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3584,0.01129919985930125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,6144,0.016976000865300496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1536,0.007255466779073079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,1024,0.006222933530807495
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,3072,0.010283733407656353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2560,0.009500799576441447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,2048,0.00817493349313736
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,768,0.005779199798901876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,512,0.005465599894523621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,256,0.004991999765237173
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,2048,128,0.004699733356634776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,16384,0.03069973389307658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,12288,0.0240447998046875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,10240,0.02086720069249471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,65536,0.11551573276519775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,8192,0.018292266130447387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,7168,0.01596799989541372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,6144,0.014990933736165366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,5120,0.012556800246238708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,4096,0.010648533701896667
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3584,0.009829333424568177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2560,0.008246399958928426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1536,0.006605866551399231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,3072,0.008854400118192036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,512,0.005148800214131674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,2048,0.007167999943097432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,1024,0.005764266848564148
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,256,0.004890666902065277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,768,0.0054666668176651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,10240,0.017259732882181803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1536,128,0.004611200094223023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,65536,0.091539200146993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,12288,0.01978879968325297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,16384,0.02486613392829895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,7168,0.013562666376431784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,8192,0.015339733163515726
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,6144,0.011428266763687134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3072,0.00766186664501826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,5120,0.00999679962793986
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,4096,0.008706133564313252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,3584,0.008501332998275758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2048,0.006459733347098033
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1536,0.006071466704209646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,1024,0.005365333457787832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,2560,0.0070602665344874065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,128,0.0045962666471799215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,512,0.005005866785844167
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,768,0.005116799970467886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,1024,256,0.004740266501903534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,65536,0.0783786694208781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,12288,0.017594667275746663
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,10240,0.015559466679890952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,8192,0.012839466333389282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,16384,0.021642667055130006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3584,0.008109866579373678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,4096,0.008307200173536937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,7168,0.011557333668073018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,6144,0.01032319962978363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,5120,0.009432533383369445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2048,0.006277333199977875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,3072,0.007382399837176005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,2560,0.006958933174610138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1536,0.005929600199063619
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,512,0.004913066824277243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,1024,0.005253333350022634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,768,0.005085866649945577
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,256,0.004645333190759023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,16384,0.021310933430989585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,12288,0.016497066617012023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,65536,0.07110826969146729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,768,128,0.004556799928347269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,8192,0.012211199601491292
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,5120,0.009327999750773112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,10240,0.014333867033322654
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,7168,0.011327999830245971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,6144,0.010289067029953003
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,4096,0.008317866424719492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3584,0.008089600006739299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2560,0.006941866874694824
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,3072,0.007287466526031494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,2048,0.006291200220584869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,512,0.004907733201980591
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1536,0.005869866907596588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,1024,0.00525439977645874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,128,0.004424533247947693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,768,0.005036800106366476
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,16384,0.020402133464813232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,65536,0.07030506928761801
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,512,256,0.004576000074545542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,12288,0.01607039968172709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,10240,0.01418773333231608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,7168,0.011340799927711486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,6144,0.010264533758163451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,5120,0.009307733178138733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,4096,0.008282666901747386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,8192,0.012227200468381246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1536,0.00584853341182073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3072,0.0072618668278058365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,3584,0.007998933394749958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2560,0.006868266562620799
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,2048,0.006187733511130015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,256,0.004599466423193614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,1024,0.005191466708978017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,768,0.0050016000866889955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,512,0.004782933493455251
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,65536,0.07065813541412354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,256,128,0.004378666480382284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,10240,0.014088533322016396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,7168,0.011222400267918905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,16384,0.020431999365488687
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,6144,0.010249599814414978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,12288,0.016054399808247886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,8192,0.012265599767367045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,4096,0.0082997332016627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,5120,0.00925973355770111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3584,0.00808426688114802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,3072,0.007242666681607564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1536,0.005890133480230967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2560,0.006925866504510244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,2048,0.006211199859778086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,128,0.004437333345413208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,1024,0.005167999863624572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,768,0.004980266590913137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,256,0.004586666822433472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,32,128,512,0.004756266872088114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,7168,0.15854612986246747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,8192,0.1793706734975179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,10240,0.22661226590474448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,12288,0.2671488126118978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,6144,0.13775466283162435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,4096,0.09555520216623942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,5120,0.11626666386922199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3584,0.08769386609395345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,16384,0.35338239669799804
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,3072,0.07413012981414795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1536,0.04110613266626994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,1024,0.03311040004094441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2048,0.05256213347117106
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,512,0.018838399648666383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,2560,0.0628544012705485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,768,0.024272000789642333
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,128,0.01179200013478597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,65536,256,0.013853866855303446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,12288,0.0722656011581421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,10240,0.06128426790237427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,8192,0.050054399172465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,16384,0.09350612958272299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,7168,0.044956799348195395
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,6144,0.03951253493626912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,5120,0.03414719899495443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,4096,0.028716800610224406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3072,0.023291732867558798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,3584,0.02629973292350769
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2560,0.02071040074030558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,2048,0.017296000321706136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1024,0.011351466178894043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,1536,0.014311466614405313
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,768,0.009998933474222819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,512,0.008505599697430928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,128,0.005789866546789805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,256,0.006442666550477346
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,12288,0.05708479881286621
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,16384,65536,0.34948800404866537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,16384,0.07650240262349448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,10240,0.04838506778081258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,5120,0.027154133717219038
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,8192,0.039527467886606854
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,6144,0.03155413269996643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,7168,0.03991573254267375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2560,0.016286933422088624
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3584,0.020900267362594604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,4096,0.022924800713857017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,3072,0.018634666999181114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1024,0.009641599655151368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,2048,0.013906133174896241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,1536,0.011612799763679505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,512,0.0070592001080513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,768,0.008589866757392883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,256,0.0057205334305763245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,128,0.005128533144791921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,12288,65536,0.2706517219543457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,16384,0.07058453559875488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,12288,0.05183360179265341
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,10240,0.04413546721140544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,8192,0.03863786856333415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,7168,0.03324693242708842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,6144,0.028411734104156493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3072,0.01692906618118286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,4096,0.02074026664098104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,3584,0.018757333358128868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2560,0.014706133802731832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,5120,0.02439146637916565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1536,0.010687999924023946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,2048,0.01256106694539388
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,1024,0.0088128000497818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,768,0.007390933235486348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,512,0.0063178668419520065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,65536,0.2348426659901937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,256,0.00554666668176651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,10240,128,0.0050122668345769245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,12288,0.04626026550928752
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,16384,0.059154133001963295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,8192,0.031691733996073404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,10240,0.037435734272003175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,7168,0.027433600028355914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,4096,0.018151466051737467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3584,0.016809600591659545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,6144,0.025093332926432295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,3072,0.014716800053914389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,5120,0.02190933426221212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2560,0.013352533181508383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,2048,0.01178666651248932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,768,0.007108266651630402
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,65536,0.19370026588439943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1536,0.01002239982287089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,1024,0.008167466521263123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,128,0.005250133574008942
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,512,0.006332799792289734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,8192,256,0.00556160012880961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,12288,0.04521919886271159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,10240,0.04036159912745158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,16384,0.06257493495941162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,8192,0.031010133028030396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,7168,0.027924267450968425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,4096,0.018232532342274985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3584,0.016544000307718913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,5120,0.022309333086013794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,6144,0.02590186595916748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2560,0.013102933764457703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,3072,0.014802133043607077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1536,0.009636267026265462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,2048,0.011394133170445759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,65536,0.20117546717325846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,1024,0.007648000121116638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,512,0.0059562668204307554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,128,0.004885333279768625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,768,0.00673173318306605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,7168,256,0.005312000215053558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,12288,0.04375360012054443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,10240,0.036892799536387126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,16384,0.060227199395497644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,8192,0.03160640001296997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,7168,0.027284266551335652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,6144,0.024380799134572348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,5120,0.02144533395767212
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,4096,0.01765120029449463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3584,0.01590720017751058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,3072,0.014430933197339377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2560,0.012982400258382163
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,65536,0.1956768035888672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,2048,0.011076266566912334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1536,0.009314133723576864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,1024,0.0072543998559316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,512,0.005985066791375478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,128,0.004833066463470459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,768,0.006538666784763336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,6144,256,0.00521066685517629
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,16384,0.05560106833775839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,10240,0.03666773239771525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,12288,0.042071465651194254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,8192,0.030482133229573567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,5120,0.019618133703867592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,7168,0.02661120096842448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,6144,0.02408426602681478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2560,0.012594133615493774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,4096,0.01710933248202006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3584,0.015172266960144043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,65536,0.18929173151652018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,3072,0.013920000195503235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,2048,0.010771200060844421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1536,0.008750933408737182
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,768,0.006482133269309997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,256,0.005151999990145365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,1024,0.007202133536338806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,128,0.004750933249791463
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,5120,512,0.005783466498057047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,12288,0.0352512001991272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,8192,0.024701867500940958
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,16384,0.05054080088933309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,10240,0.033122134208679196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,7168,0.02248319983482361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,6144,0.02041813333829244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,5120,0.018311466773351034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,65536,0.15842347145080565
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,4096,0.015757866700490317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3584,0.014194132884343466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2560,0.011813333630561829
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,3072,0.012733866771062216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1024,0.006876799960931141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,512,0.005846400062243144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,2048,0.009891200065612792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,1536,0.008270933230717977
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,128,0.004805333415667216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,768,0.006433066725730896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,4096,256,0.005234133203824361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,12288,0.03376213312149048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,10240,0.030396799246470135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,16384,0.04649386803309123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,7168,0.021771732966105142
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,8192,0.024269866943359374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,5120,0.01763946612675985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,6144,0.020478934049606323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,65536,0.1499519983927409
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,4096,0.014454399545987448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3584,0.013547733426094055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,3072,0.01241386632124583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2048,0.009537067015965779
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,2560,0.010900266965230306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,768,0.006090666850407918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1536,0.007656533519426982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,1024,0.006693333387374878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,512,0.0056991999348004665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,256,0.005143466591835022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3584,128,0.004820266862710317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,10240,0.03318079908688863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,16384,0.05108480056126913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,12288,0.03864106734593709
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,8192,0.027804799874623615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,6144,0.022232532501220703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,7168,0.025171200434366863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3584,0.014298666516939798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,5120,0.018548266092936198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,65536,0.17239999771118164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,4096,0.01578133304913839
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2560,0.011684266726175944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,2048,0.009591466188430786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1536,0.007952000200748443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,3072,0.013205333550771078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,512,0.005568000177542368
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,128,0.0046965335806210835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,1024,0.006885333359241486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,768,0.0061471998691558834
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,3072,256,0.0049792001644770306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,10240,0.029499733448028566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,8192,0.024781866868336996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,16384,0.047048532962799074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,12288,0.03501226504643758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,7168,0.022453333934148154
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,6144,0.020092799266179403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,5120,0.01737706661224365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,4096,0.014446933070818582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2048,0.008553600311279297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,65536,0.15754559834798176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3584,0.013180800278981528
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,3072,0.01144533356030782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,2560,0.010264533758163451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1536,0.0073845331867535904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,768,0.005721599857012431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,512,0.005426133175690969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,128,0.00468800018231074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,256,0.004961066444714864
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2560,1024,0.006457599997520447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,12288,0.035767467816670735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,10240,0.03033706744511922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,16384,0.04397653341293335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,8192,0.023162666956583658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,65536,0.14618773460388185
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,6144,0.02083946665128072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,7168,0.02333866755167643
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3584,0.01302079955736796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,3072,0.011905066172281901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,5120,0.016054399808247886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,4096,0.014613333344459533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2048,0.009058133761088053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,2560,0.010149332880973815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,768,0.006178133189678192
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1024,0.006523733337720235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,512,0.005677866439024607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,1536,0.007669333120187123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,12288,0.029215999444325763
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,128,0.004693333307902018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,2048,256,0.0050122668345769245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,16384,0.03773973385492961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,10240,0.02534293333689372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,65536,0.12773866653442384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,8192,0.021310933430989585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,7168,0.02008533279101054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,6144,0.017439999183019004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,5120,0.014660267035166421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,4096,0.012012799580891926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3072,0.010284800330797832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2560,0.009157333771387737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,2048,0.007877333462238312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,3584,0.010892800490061442
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,768,0.005624533196290334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1536,0.006794666747252147
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,128,0.004680533210436503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,1024,0.0059690664211908976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,512,0.005303466816743215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1536,256,0.0048426667849222815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,65536,0.1006719986597697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,16384,0.030103466908137005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,12288,0.023206400871276855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,10240,0.019088000059127808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,8192,0.017112533251444496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,4096,0.009257599711418152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3584,0.00879253347714742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,7168,0.015310933192571005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,6144,0.0132832000652949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,5120,0.011386666695276897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2560,0.007477333148320516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,768,0.005309866865475973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,2048,0.006705066561698914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1536,0.006151466568311056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,3072,0.007994666695594788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,1024,0.005480533341566721
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,256,0.0047082667549451195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,512,0.0049792001644770306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,1024,128,0.004589866598447164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,16384,0.02442986567815145
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,65536,0.08474986553192139
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,12288,0.01952106753985087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,7168,0.012025599678357441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,10240,0.016862932840983072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,6144,0.010503466924031575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,8192,0.013300266861915589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,4096,0.008427733182907104
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3072,0.00721919983625412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,5120,0.009426132837931315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,3584,0.008086400230725606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2560,0.006994133194287618
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1024,0.005410133302211762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,768,0.005049600203831991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,1536,0.005885866781075796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,2048,0.006258133550484974
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,512,0.004886400202910105
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,256,0.004637866715590159
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,16384,0.020797866582870483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,12288,0.01621333360671997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,10240,0.014084266622861228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,65536,0.06935253143310546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,768,128,0.004429866870244345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,7168,0.011159466703732808
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,8192,0.012116266290346782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,6144,0.010060800115267436
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,4096,0.008125866452852886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3584,0.007962666451931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2560,0.006902400155862172
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,5120,0.009192533294359843
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,2048,0.006197333335876465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,3072,0.007129600147406261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,768,0.005008000135421753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1024,0.005193600058555603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,1536,0.005826133489608765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,128,0.004454400142033895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,512,0.0047541335225105286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,16384,0.019808000326156615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,512,256,0.004572799801826477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,65536,0.0683679978052775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,10240,0.013969066739082336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,8192,0.012062933047612507
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,12288,0.01588159998257955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,7168,0.01104853351910909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,4096,0.008108800152937572
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3584,0.007868800063927968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,3072,0.007077333331108093
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2560,0.006841599941253662
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,6144,0.010024533669153849
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,5120,0.009044266740481059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1024,0.005201066533724466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,2048,0.006161066889762879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,1536,0.005832533538341522
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,512,0.004747733473777771
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,768,0.004962133367856344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,128,0.0044170667727788295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,256,256,0.004533333579699198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,8192,0.012012799580891926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,16384,0.019792000452677407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,65536,0.06872320175170898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,12288,0.01583999991416931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,10240,0.013942399621009826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,5120,0.009083732962608337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3584,0.007949866851170858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,7168,0.011068800091743469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,6144,0.010003200173377991
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2560,0.006885333359241486
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,4096,0.008027733365694682
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,2048,0.0061258668700853985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1536,0.00581333339214325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,1024,0.005161599814891815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,3072,0.007000533243020375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,768,0.004971733192602793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,512,0.004691199958324432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,256,0.004496000210444133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,16,128,128,0.00436160018046697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,7168,0.1586858590443929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,8192,0.17796907424926758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,12288,0.2659584045410156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,10240,0.22488320668538414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,16384,0.36091839472452797
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,5120,0.11491519610087078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3584,0.08705386320749918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,6144,0.1378346602121989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,4096,0.1011840025583903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1024,0.032358400026957196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2560,0.06405760049819946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,1536,0.04172799984614055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,2048,0.05192213455835978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,3072,0.08291412989298502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,768,0.023849600553512575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,128,0.011754666765530903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,256,0.013717333475748697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,65536,512,0.018463999032974243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,8192,0.049814399083455405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,10240,0.06133120059967041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,16384,0.09233813285827637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,12288,0.0714143991470337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,7168,0.04435733159383138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,5120,0.03353600104649861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,4096,0.02834133307139079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,6144,0.039331201712290445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3584,0.025997867186864216
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,3072,0.022954666614532472
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2560,0.02041813333829244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1536,0.014259200294812521
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,2048,0.017130666971206666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,1024,0.011213866869608562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,512,0.008364799618721008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,768,0.010083199540774027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,256,0.0064629331231117245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,128,0.005694933235645294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,12288,0.059495464960734046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,16384,0.07341866493225098
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,10240,0.04901226758956909
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,16384,65536,0.3600672086079916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,7168,0.03616426785786946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,8192,0.04166826804478963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,6144,0.03202026685078939
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,5120,0.0301418662071228
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,4096,0.02327573299407959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2560,0.016992000738779704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3584,0.02156053384145101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,3072,0.01881493330001831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,2048,0.014252799749374389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1536,0.011646933356920878
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,1024,0.00962666670481364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,768,0.008657067020734151
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,512,0.006726400057474773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,256,0.005819733440876007
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,128,0.005148800214131674
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,12288,65536,0.2742848078409831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,12288,0.05551679929097494
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,16384,0.06755092938741049
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,10240,0.04737279812494914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,8192,0.03909866809844971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,6144,0.029139200846354168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,7168,0.03460799853006999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,5120,0.02620159983634949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,4096,0.02167466680208842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3072,0.017449599504470826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2048,0.013060266772905985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,3584,0.01967039903004964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1024,0.008798933029174805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,2560,0.015460266669591268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,512,0.00631039987007777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,1536,0.010923733313878376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,768,0.008044800162315369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,65536,0.2427061398824056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,256,0.005553066730499268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,10240,128,0.004915200173854828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,12288,0.046615465482076006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,16384,0.05686080058415731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,10240,0.03895999987920125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,8192,0.030376533667246502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,7168,0.0273632009824117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,6144,0.024158932765324912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,5120,0.022146133581797282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,4096,0.01818880041440328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3072,0.014745600024859109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1536,0.010168533523877461
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,3584,0.016812799374262492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,65536,0.20527359644571938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2048,0.011553066968917846
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,2560,0.013639466961224875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,1024,0.008329600095748901
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,768,0.0073183998465538025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,512,0.006470400094985962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,128,0.005222400029500326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,8192,256,0.005634133517742157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,8192,0.03301440080006917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,10240,0.043137065569559735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,12288,0.05002880096435547
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,16384,0.06096320152282715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,6144,0.028115199009577437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,5120,0.023753599325815836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,4096,0.019598933060963948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,7168,0.029772800207138062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3072,0.015794133146603904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2560,0.013867732882499696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,3584,0.017997866868972777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,2048,0.01200213332970937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1024,0.007879466811815897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,512,0.00595413347085317
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,1536,0.009994666775067646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,65536,0.21282025973002114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,256,0.005329066514968872
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,768,0.006926933427651723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,7168,128,0.004874666531880697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,8192,0.031804800033569336
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,10240,0.0419050653775533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,12288,0.04524373213450114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,16384,0.058840533097585045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,5120,0.021699200073877968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,4096,0.018602667252222697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,7168,0.028707200288772584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,6144,0.02722240090370178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2560,0.013589333494504294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3584,0.016989866892496742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,3072,0.015184000134468079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,2048,0.011448533336321513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,65536,0.21107519467671715
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1536,0.009777067104975383
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,1024,0.007470933099587758
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,768,0.006903466582298279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,256,0.005298133194446564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,512,0.005955199897289276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,6144,128,0.004806399842103322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,16384,0.05684266487757365
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,10240,0.04024853308995564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,12288,0.046562135219573975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,8192,0.030781867106755574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,6144,0.026295467217763262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,7168,0.02752853234608968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,5120,0.021963733434677123
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3072,0.014922666549682616
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,4096,0.018268799781799315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,3584,0.016952532529830932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2048,0.011364266276359558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,65536,0.20068693161010742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,2560,0.01339413324991862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,768,0.0067669332027435304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1536,0.009398399790128072
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,512,0.005868799984455109
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,1024,0.007300266623497009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,256,0.005102933446566264
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,5120,128,0.00476800004641215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,12288,0.03905066649119059
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,10240,0.033869866530100504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,7168,0.02446933388710022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,16384,0.04696640173594157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,5120,0.018628267447153728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,4096,0.015135999520619711
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,8192,0.02525866627693176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,6144,0.021657600005467733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,65536,0.16549439430236818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3584,0.014505599935849508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,3072,0.01267413298288981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2048,0.010150399804115296
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,2560,0.011658666531244914
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,256,0.005310933291912079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,768,0.006666666766007741
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1536,0.008374399940172831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,1024,0.007002666592597961
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,512,0.005902933577696482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,10240,0.03207893371582031
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,4096,128,0.0049450665712356566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,12288,0.03739733298619588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,16384,0.04695680141448975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,6144,0.021118932962417604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,8192,0.027078400055567425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,7168,0.02276479999224345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,4096,0.015607466300328573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3584,0.014320000012715658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,5120,0.01735360026359558
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,65536,0.16072319348653158
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2048,0.009717333316802978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,3072,0.012566399574279786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,2560,0.011255466938018798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,768,0.006586666901906331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1536,0.008096000055472057
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,512,0.005783466498057047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,256,0.005166933437188466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,1024,0.006775466601053874
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3584,128,0.004822400212287903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,8192,0.030739200115203858
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,10240,0.035382401943206784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,16384,0.05327253341674805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,12288,0.043543465932210285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,7168,0.027569067478179932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,6144,0.022485333681106567
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,5120,0.01984106699625651
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3584,0.015713066856066386
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,3072,0.013970133662223817
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2560,0.012170666456222534
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,4096,0.01708586613337199
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,2048,0.010253866513570149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1024,0.006809600194295247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,768,0.006419200201829274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,65536,0.18825920422871906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,1536,0.008307200173536937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,512,0.005667200187842051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,256,0.005106133222579956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,3072,128,0.0047199999292691554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,16384,0.048162134488423665
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,10240,0.031940267483393354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,12288,0.03795093297958374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,8192,0.02773226698239644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,65536,0.17132266362508136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,6144,0.021062399943669638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,7168,0.024102399746576943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,5120,0.01883946657180786
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,4096,0.015631999572118124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3072,0.012391466895739238
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2048,0.009105066458384197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,2560,0.01125759979089101
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,3584,0.015134933590888976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,512,0.005414400001366933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1536,0.008166400094827015
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,256,0.004948266843954722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,1024,0.006504533191521962
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,768,0.006268799801667531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,12288,0.03356373310089111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2560,128,0.004605866471926371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,16384,0.04390613238016765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,10240,0.030763733386993408
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,65536,0.1690111955006917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,6144,0.019356799125671387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,8192,0.02423786719640096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,7168,0.022073600689570108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,5120,0.017808000246683754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,4096,0.014434132973353067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2048,0.009145599603652955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3584,0.012660266955693564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,3072,0.011904000242551168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,2560,0.010410666465759277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1536,0.008181333541870117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,768,0.006433066725730896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,512,0.005704533557097117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,128,0.004774400095144907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,1024,0.006593066453933716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,16384,0.0403221329053243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,2048,256,0.005109333495299021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,12288,0.03223359982172648
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,7168,0.02055466572443644
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,6144,0.018613332509994508
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,10240,0.028293333450953168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,8192,0.023181867599487305
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,65536,0.14392959276835124
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,5120,0.015381333231925965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3584,0.012089600165685017
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,3072,0.010635733604431152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,4096,0.01306773324807485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2560,0.009320533275604248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,2048,0.008136533200740814
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1536,0.007233066856861115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,1024,0.00606826643149058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,768,0.006155733267466227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,512,0.005317333340644837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,128,0.004709333181381226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1536,256,0.004930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,16384,0.032323199510574344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,12288,0.025778132677078246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,10240,0.021948800484339396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,6144,0.014288000265757241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,5120,0.012597333391507468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,65536,0.11042986710866291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,8192,0.01903146704037984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,7168,0.01663253307342529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3584,0.010550399621327717
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,3072,0.008602666854858398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2560,0.008591999610265095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,4096,0.010533333818117777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,2048,0.006903466582298279
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1536,0.006278400123119354
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,512,0.004910933474699656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,1024,0.005548800031344095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,768,0.005532800157864889
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,256,0.0047647997736930845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,1024,128,0.004582400123278299
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,65536,0.09417280356089273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,16384,0.02688960035641988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,12288,0.02082560062408447
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,7168,0.012768000364303589
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,6144,0.011490133404731751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,5120,0.009960533181826273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,10240,0.017413334051767985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,8192,0.01428053379058838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2560,0.007264000177383423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,2048,0.006412800153096516
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,4096,0.008718933661778767
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3584,0.008806399504343669
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,3072,0.007525333265463512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1536,0.005990399916966756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,1024,0.005293866495291392
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,768,0.005208533505598704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,512,0.004935466746489207
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,256,0.0046517332394917805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,12288,0.016429866353670754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,10240,0.0138154665629069
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,768,128,0.00453653335571289
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,16384,0.020777599016825358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,65536,0.07431999842325845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,6144,0.009940266609191895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,5120,0.009089066584904989
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,8192,0.011944533387819926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,7168,0.010956799983978272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,4096,0.008086400230725606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1536,0.00581226646900177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3584,0.007940266529719036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,3072,0.007117866476376851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2560,0.006851199766000111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,1024,0.005145599941412607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,2048,0.006234666705131531
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,768,0.005004799862702688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,512,0.004730666677157084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,256,0.004523733258247375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,65536,0.0678879976272583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,16384,0.019602133830388387
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,10240,0.013760000467300415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,512,128,0.004380799829959869
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,8192,0.011850666999816895
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,12288,0.015661866466204325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,6144,0.009959466258684794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,5120,0.00899733304977417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,7168,0.010974933703740437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2048,0.006141866743564606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,4096,0.008070399860541026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3584,0.007860266665617625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,3072,0.0071263998746871945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,2560,0.006795733173688252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1536,0.005799466868241628
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,768,0.004956800242265066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,1024,0.00510506679614385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,512,0.004772266745567322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,128,0.004337066908677419
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,256,256,0.004457599918047587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,12288,0.01570453345775604
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,65536,0.06810773213704427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,10240,0.013729066650072733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,7168,0.010922666390736897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,6144,0.009939199686050415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,16384,0.019554134209950766
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,8192,0.01183573305606842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,4096,0.008029866715272267
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3072,0.007049599786599477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,5120,0.008990933497746784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2048,0.006156800190607706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,2560,0.006870399912198384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,3584,0.007904000083605449
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1024,0.005144533514976501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,256,0.004446933170159658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,512,0.004682666560014089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,1536,0.005834666887919108
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,768,0.004938666522502899
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,8,128,128,0.004320000112056732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,7168,0.15755200386047363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,8192,0.17775786717732747
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,10240,0.22441066106160484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,12288,0.26872212092081704
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,5120,0.11437760194142658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,6144,0.1363210678100586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,16384,0.35320428212483723
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,4096,0.09642240206400553
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3584,0.09377919832865397
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1024,0.03221759994824727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2560,0.062320001920064295
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,3072,0.07257813612620036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,2048,0.057801600297292074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,1536,0.04270826578140259
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,256,0.014193066954612732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,768,0.023829332987467446
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,512,0.01830613414446513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,65536,128,0.01151146690050761
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,8192,0.04944426616032918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,12288,0.0715936024983724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,10240,0.06119146744410196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,16384,0.09255893230438232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,4096,0.028011733293533327
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,7168,0.044086400667826334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,6144,0.03919359842936198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,5120,0.0335647980372111
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2048,0.01698346734046936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3072,0.022807466983795165
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1024,0.01120853324731191
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,2560,0.020413867632548013
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,3584,0.026062933603922527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,768,0.010178132851918539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,1536,0.014818132917086283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,128,0.00559146652619044
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,512,0.008277333279450735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,256,0.006425599753856659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,10240,0.05081493457158407
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,12288,0.06071466604868571
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,16384,0.07836799621582032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,16384,65536,0.35615466435750326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,8192,0.04139946699142456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,7168,0.037240533034006755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,6144,0.03259733319282532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,4096,0.02395626703898112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,5120,0.03024959961573283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2560,0.018100267648696898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3072,0.019373865922292073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,3584,0.023459200064341226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,2048,0.01395840048789978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1536,0.01311360001564026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,1024,0.009590400258700053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,128,0.00496319979429245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,768,0.00906773308912913
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,512,0.006969599922498067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,256,0.005843199789524078
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,12288,65536,0.28840853373209635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,12288,0.0548746665318807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,16384,0.07496639887491861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,10240,0.04755093256632487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,8192,0.04130239884058635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,6144,0.029838933547337847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,7168,0.03385706742604573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,5120,0.026390399535497027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,4096,0.02251946727434794
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3584,0.020935465892155967
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2560,0.01688213348388672
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,3072,0.018024533987045288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1536,0.012284800410270691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,1024,0.009136000275611877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,768,0.008357333143552144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,2048,0.013353600104649862
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,512,0.0065984000762303666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,256,0.005602133274078369
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,65536,0.2553642590840658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,10240,128,0.00487253318230311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,10240,0.040593067804972335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,16384,0.06196800072987875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,12288,0.04609493414560954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,8192,0.03118293285369873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,7168,0.028051199515660603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,6144,0.025813333193461102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,5120,0.022869332631429037
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3584,0.02023680011431376
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2560,0.01581546664237976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,2048,0.011664000153541566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,4096,0.01851093371709188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,3072,0.015245866775512696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1536,0.012039466698964437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,1024,0.008620799581209818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,768,0.007768533130486806
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,256,0.005780266722043356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,512,0.006734933455785115
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,65536,0.2184234619140625
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,8192,128,0.00513919989267985
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,12288,0.052457598845163975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,16384,0.06761173407236734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,10240,0.04534613291422526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,8192,0.03564373254776001
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,7168,0.03161919911702474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,4096,0.01990506649017334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,5120,0.024001065889994302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3584,0.019818667570749918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,3072,0.01664959987004598
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2560,0.015560533603032431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,6144,0.027628799279530842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1536,0.011405866344769795
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,2048,0.012486400206883748
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,768,0.007216000060240428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,512,0.006247466802597046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,256,0.005390933156013489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,128,0.0047199999292691554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,1024,0.008110933502515157
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,7168,65536,0.22786026000976561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,10240,0.044223999977111815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,12288,0.05063039859135946
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,16384,0.06751893361409506
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,5120,0.022997333606084188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,7168,0.03099199930826823
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,6144,0.026947200298309326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,8192,0.03533759911855062
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,4096,0.019819732507069907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3072,0.01514240006605784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,3584,0.01939520041147868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2560,0.015169066190719605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,2048,0.012020267049471537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1536,0.011068800091743469
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,1024,0.007817600170771282
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,65536,0.2240074634552002
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,256,0.005373866856098175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,128,0.004786133269468943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,768,0.007255466779073079
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,6144,512,0.006196266909440359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,10240,0.04194453159968058
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,16384,0.06248533328374227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,8192,0.03594453334808349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,12288,0.04911253452301025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,6144,0.025937066475550337
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,5120,0.022194133202234904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,7168,0.029020800193150835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,4096,0.019751467307408652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3584,0.017352533340454102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2048,0.011653332908948263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,65536,0.21555627187093102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,2560,0.014776532848676046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,3072,0.015309866269429526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1536,0.01072746713956197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,1024,0.0075989335775375364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,512,0.006105599800745646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,768,0.006995200117429097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,256,0.005234133203824361
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,5120,128,0.0046965335806210835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,10240,0.03472533226013184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,12288,0.03814613421758016
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,8192,0.027035733064015705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,16384,0.05088106791178385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,7168,0.02449493408203125
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,6144,0.021398399273554483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,4096,0.01548799971739451
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2560,0.013967999815940857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,5120,0.01949440042177836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3584,0.015929599603017174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1536,0.01004266639550527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,3072,0.013962666193644205
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,65536,0.19124053319295248
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,1024,0.007301333546638489
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,768,0.007133866846561432
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,512,0.006155733267466227
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,2048,0.010382933417956035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,256,0.005364266534646353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,4096,128,0.004760533571243286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,12288,0.03800533215204875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,16384,0.05068373282750448
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,10240,0.034378667672475174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,8192,0.028578132390975952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,7168,0.022858667373657226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,6144,0.02129813234011332
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,5120,0.01915093262990316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,4096,0.016194132963816325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3072,0.013722667098045349
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2560,0.013053866227467856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,65536,0.1721706708272298
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,3584,0.01625386675198873
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,2048,0.009853866696357728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1536,0.009477333227793375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,1024,0.007156266768773396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,512,0.005998933315277099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,768,0.006923733154932658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,128,0.004673066735267639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3584,256,0.005267199873924255
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,12288,0.047635201613108316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,8192,0.03408533334732056
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,16384,0.060779734452565515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,10240,0.039774934450785324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,65536,0.2069002628326416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,7168,0.026949334144592284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,4096,0.018268799781799315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,6144,0.02600319981575012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,5120,0.02059733271598816
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2048,0.010665599505106609
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3584,0.01722773313522339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1536,0.009556266665458679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,3072,0.015118933717409768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,2560,0.013788800438245138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,1024,0.0072970668474833175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,768,0.00699946681658427
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,256,0.005178666611512502
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,128,0.004565333326657614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,3072,512,0.0060032000144322716
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,10240,0.035197865962982175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,12288,0.04192853371302287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,16384,0.056109865506490074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,8192,0.029785599311192828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,6144,0.02292693257331848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,5120,0.02009493311246236
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,65536,0.18428373336791992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3584,0.015387733777364096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,7168,0.024516266584396363
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,3072,0.014570666352907815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,4096,0.016660267114639284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2048,0.009435733159383137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1536,0.008843732873598735
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,2560,0.012666666507720947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,256,0.004981333514054617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,1024,0.007097599903742473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,768,0.00652266691128413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,128,0.004574933151404063
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2560,512,0.005667200187842051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,12288,0.03561813433965047
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,10240,0.03171306649843852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,8192,0.025967999299367266
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,16384,0.049697065353393556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,5120,0.017884800831476845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,7168,0.021539199352264404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,4096,0.014862933754920959
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3584,0.014134400089581809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,65536,0.17034559249877929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,6144,0.021022933721542358
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2560,0.012134400010108948
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,2048,0.009143466750780743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,3072,0.012516267100969949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1024,0.007048533360163371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,512,0.005983999868233999
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,1536,0.009526399771372478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,256,0.005217066903909048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,128,0.004694400231043497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,2048,768,0.007052800059318543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,16384,0.04386986494064331
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,12288,0.03395413160324097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,10240,0.02977493405342102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,8192,0.024995199839274087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,7168,0.020166399081548055
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,6144,0.0191210667292277
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,65536,0.15147412618001302
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,5120,0.016872533162434897
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3584,0.013269333044687906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,4096,0.013502933581670127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2560,0.010710400342941285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,3072,0.011576533317565918
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1536,0.008342400193214417
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,1024,0.006611200173695882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,768,0.006392533580462138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,2048,0.008500267068545024
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,256,0.004930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,128,0.004584533472855886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1536,512,0.005593599875768026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,12288,0.027827199300130206
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,16384,0.03573226531346639
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,65536,0.11300160090128582
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,10240,0.02395626703898112
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,8192,0.020983467499415077
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,7168,0.01684373418490092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,5120,0.013267200191815695
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,4096,0.011162666479746501
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3072,0.010259200135866802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,6144,0.01588159998257955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,3584,0.011993599931399028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2048,0.007416533430417378
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1024,0.005817600091298421
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,2560,0.009725866715113322
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,256,0.004818133513132731
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,768,0.005866666634877523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,1536,0.007457066575686138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,512,0.005156266689300537
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,1024,128,0.004548266530036926
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,10240,0.018503467241923012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,8192,0.015044266978899637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,65536,0.10363626480102539
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,12288,0.022694400946299233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,16384,0.02901653250058492
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,6144,0.012167466680208842
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,5120,0.011669333775838215
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,4096,0.009433600306510925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3584,0.010407466689745586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,7168,0.012785067160924276
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,3072,0.007881600161393483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2048,0.006520533561706543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1536,0.0067775999506314594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,768,0.005502933263778686
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,1024,0.005420800050099691
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,2560,0.008461866776148479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,128,0.004448000093301137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,256,0.004658133288224538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,16384,0.021618133783340453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,65536,0.07817493279774984
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,768,512,0.004934399823347728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,12288,0.01708266735076904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,10240,0.01378666659196218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,8192,0.011981866757074992
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,7168,0.010914132992426554
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,5120,0.009177600344022114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3584,0.007921066880226136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2560,0.006930133203665416
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,3072,0.0071050668756167095
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,6144,0.010012800494829815
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,4096,0.008074666559696197
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1536,0.00588266650835673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,768,0.0050687998533248905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,512,0.0047775998711586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,256,0.004524800181388855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,1024,0.005275733272234599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,2048,0.006077866752942403
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,65536,0.067303466796875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,16384,0.019495467344919838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,12288,0.015608533223470052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,512,128,0.004408533374468485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,8192,0.011829333504041036
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,6144,0.009885866443316143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,5120,0.009008000294367473
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,10240,0.013739732901255288
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,4096,0.008019199967384339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,7168,0.010875733693440755
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3584,0.007854933540026348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,3072,0.007121066749095917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2560,0.006780800223350525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,2048,0.006052266558011373
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1536,0.005827199916044871
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,768,0.004924799998601278
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,512,0.0047082667549451195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,256,0.0044608001907666525
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,128,0.004345599810282389
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,256,1024,0.0051584000388781226
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,65536,0.06756479740142822
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,16384,0.019385600090026857
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,12288,0.01557866632938385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,7168,0.010860799749692281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,8192,0.011876266201337178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,6144,0.009849599997202555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,5120,0.009015466769536335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,10240,0.013677866260210673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,4096,0.00802346666653951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3072,0.007048533360163371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,3584,0.00796693315108617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2560,0.006796800096829732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,2048,0.006067200005054474
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1536,0.0057429333527882894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,1024,0.005098666747411092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,768,0.004955733319123586
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,256,0.004446933170159658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,512,0.004690133531888326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,4,128,128,0.004242133100827535
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,7168,0.15679359436035156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,8192,0.17972373962402344
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,10240,0.22581547101338706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,12288,0.27306346893310546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,6144,0.13684372901916503
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,5120,0.11416213512420655
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,16384,0.35780693689982096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,4096,0.09617173671722412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2048,0.05739519993464152
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,2560,0.06311786572138468
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3584,0.09117013613382975
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,3072,0.07177279790242513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1536,0.03944853146870931
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,256,0.013736533125241599
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,1024,0.029121067126591998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,512,0.019692800442377725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,768,0.025333333015441894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,65536,128,0.01164906620979309
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,8192,0.04913493394851685
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,10240,0.060805332660675046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,12288,0.07213226954142252
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,16384,0.09251093069712321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,6144,0.0387722651163737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,5120,0.03338346481323242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,7168,0.04394986629486084
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3072,0.022593067089716593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,3584,0.025568000475565594
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,4096,0.02788693308830261
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2560,0.020246400435765585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1536,0.014255999525388082
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,1024,0.011221333344777425
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,768,0.010494933525721232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,2048,0.01700053413709005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,512,0.008297599852085114
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,128,0.005580799778302511
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,256,0.006637866795063019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,10240,0.05084799925486246
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,16384,0.07640426953633626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,12288,0.06049173275629679
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,16384,65536,0.356169605255127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,7168,0.03599040110905965
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,6144,0.032484267155329385
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,8192,0.043397335211435954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,5120,0.030349866549173994
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,4096,0.023694932460784912
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2560,0.01638826628526052
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,2048,0.013843199610710144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3584,0.021593600511550903
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,3072,0.01911146640777588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,512,0.006850133339564006
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1536,0.01195733348528544
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,768,0.00955839951833089
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,1024,0.009573333462079366
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,256,0.005962666869163513
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,65536,0.2904853185017904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,12288,128,0.004946133494377137
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,8192,0.04011733531951904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,10240,0.050316798686981204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,12288,0.05702826579411825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,16384,0.07412160237630208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,5120,0.025498666365941364
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,7168,0.035469865798950194
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,6144,0.03030186692873637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,4096,0.02190613349278768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3584,0.020035199324289956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1536,0.011252267162005107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2048,0.013479466239611307
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,2560,0.01548373301823934
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,1024,0.00897173285484314
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,3072,0.016925867398579916
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,768,0.009013332923253377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,512,0.006596266726652781
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,256,0.005740800003210703
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,128,0.00477866679430008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,10240,65536,0.256277338663737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,10240,0.037973332405090335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,16384,0.06057386795679728
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,12288,0.0465770681699117
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,8192,0.03619199991226196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,6144,0.02553706765174866
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,7168,0.029151999950408937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,5120,0.022227199872334798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3072,0.014814933141072592
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,4096,0.01851946711540222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,3584,0.017093332608540852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2048,0.011815466483434041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1536,0.010500267148017883
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,2560,0.013758933544158936
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,768,0.008188800017038981
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,256,0.00592853327592214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,128,0.005043200155099233
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,65536,0.21101333300272623
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,512,0.00664213349421819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,8192,1024,0.008710400263468424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,8192,0.03751360177993775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,12288,0.05166399876276652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,10240,0.04157013495763143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,16384,0.06644479831059774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,7168,0.032731733719507855
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,6144,0.027532800038655596
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,4096,0.020091732343037925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,5120,0.02242986758550008
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3584,0.01834133267402649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1536,0.01042133371035258
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,3072,0.014967466394106546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2560,0.013760000467300415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,1024,0.008098133405049642
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,2048,0.012410666545232136
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,512,0.006247466802597046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,256,0.005593599875768026
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,65536,0.23670825958251954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,128,0.004711466530958811
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,7168,768,0.007820799946784973
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,12288,0.052571733792622886
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,8192,0.035922133922576906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,16384,0.06922667026519776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,10240,0.04007999897003174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,5120,0.022510933876037597
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,4096,0.019562667608261107
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3584,0.017987199624379478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,7168,0.03218986590703328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,6144,0.02621120015780131
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,3072,0.015149866541226705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2560,0.014223999778429666
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,2048,0.012157866358757019
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,768,0.007841066519419352
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,512,0.006167466441790262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,65536,0.22878079414367675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1536,0.010099200407663982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,1024,0.007609599828720092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,256,0.005577600002288819
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,6144,128,0.004645333190759023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,12288,0.050536533196767174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,10240,0.040880000591278075
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,16384,0.06286826531092325
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,6144,0.02611306707064311
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,5120,0.020177066326141357
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,8192,0.035905067125956217
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,7168,0.02792853315671285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,4096,0.019361066818237304
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3584,0.01684266726175944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2560,0.01260693371295929
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,3072,0.01590079963207245
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1536,0.00974826713403066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,1024,0.007472000022729237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,2048,0.01186346709728241
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,65536,0.2187061309814453
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,768,0.007666133344173431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,512,0.006074666480223338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,256,0.005514666438102722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,5120,128,0.004539733131726583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,16384,0.05301546653111776
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,10240,0.03163413405418396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,7168,0.025748266776402788
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,6144,0.0201855997244517
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,12288,0.03839786847432454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,5120,0.018076799313227334
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,4096,0.015585066874821982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3584,0.013667200009028116
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,8192,0.029641600449879964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,65536,0.18340160051981608
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,3072,0.012943999965985617
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2560,0.011276800433794658
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,2048,0.010263466835021972
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1536,0.008941866954167684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,1024,0.0072629332542419435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,768,0.007773866752783458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,128,0.004670933385690053
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,512,0.006166400015354156
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,4096,256,0.005541333556175232
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,16384,0.05330453316370646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,12288,0.039442133903503415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,8192,0.028376533587773638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,7168,0.025042132536570234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,6144,0.021052799622217813
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,5120,0.016682666540145875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,4096,0.01616213321685791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,10240,0.0307370662689209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3584,0.01476479967435201
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,65536,0.17504000663757324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,3072,0.012326400478680928
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1536,0.008679466446240743
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2560,0.011895466844240825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,768,0.007225599884986877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,2048,0.009902933239936828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,1024,0.007165866593519847
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,256,0.005402666827042898
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,128,0.004658133288224538
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3584,512,0.005970133344332377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,10240,0.037112534046173096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,16384,0.059432534376780186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,7168,0.029051733016967774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,12288,0.047993600368499756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,8192,0.034253867467244466
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,5120,0.01871573328971863
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,6144,0.024247467517852783
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,65536,0.20138773918151856
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,4096,0.018295466899871826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3584,0.01663146714369456
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,3072,0.013126400113105775
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2560,0.012312533458073933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,2048,0.010955733060836793
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,768,0.007561600208282471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,512,0.005810133119424184
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,256,0.005410133302211762
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1536,0.008824533224105835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,128,0.004539733131726583
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,3072,1024,0.007141333321730297
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,12288,0.042931199073791504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,10240,0.0337226668993632
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,8192,0.03206826647122701
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,6144,0.021758933862050377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,16384,0.05484480063120524
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,5120,0.018500266472498576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,7168,0.02624746759732564
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,65536,0.18814187049865722
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,4096,0.01581439971923828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3584,0.014940800269444785
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,3072,0.013753599921862283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2560,0.011532800396283467
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1024,0.006855466465155284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,768,0.007000533243020375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,2048,0.009423999985059103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,1536,0.008444799979527792
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,256,0.005188266436258951
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,128,0.004523733258247375
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2560,512,0.005656533439954122
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,12288,0.0364522655804952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,10240,0.030640000104904176
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,16384,0.048078934351603195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,8192,0.0288480003674825
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,6144,0.019577600558598838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,5120,0.015955199797948204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,65536,0.1712970733642578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,4096,0.014532267053922018
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,7168,0.02190613349278768
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2560,0.010301867127418518
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3072,0.010957866907119751
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,2048,0.009170132875442504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,3584,0.013038933277130127
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,768,0.007595733304818471
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1536,0.008088533580303193
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,256,0.005430399874846141
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,1024,0.007047466437021891
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,128,0.0045855998992919925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,2048,512,0.005967999994754791
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,8192,0.02709973255793254
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,16384,0.04657919804255168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,65536,0.15186559359232585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,12288,0.033187200625737504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,10240,0.027407999833424884
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,7168,0.02197973330815633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,6144,0.018272000551223754
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,4096,0.012243200341860454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3072,0.01111893355846405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2048,0.008455466230710347
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,5120,0.015074132879575094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,3584,0.011380267143249512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1536,0.007764266431331634
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,2560,0.00977280040582021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,768,0.006545066833496094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,512,0.005549866457780202
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,128,0.004519466559092204
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,1024,0.00652266691128413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,16384,0.03586560090382894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1536,256,0.0051125332713127134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,65536,0.11875306765238444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,8192,0.022002132733662923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,7168,0.017666133244832356
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,5120,0.01193386713663737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,6144,0.014150399963061014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,10240,0.022992000977198283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,12288,0.027911466360092164
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,4096,0.011218133568763732
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3072,0.009276800354321798
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,3584,0.010014933347702027
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2560,0.008144000172615051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,2048,0.00713919997215271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1536,0.0066442668437957765
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,1024,0.005818666517734527
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,768,0.005854933460553487
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,512,0.005097599824269613
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,256,0.004786133269468943
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,1024,128,0.004424533247947693
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,16384,0.02959573268890381
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,12288,0.02299626668294271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,10240,0.01800640026728312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,8192,0.018517333269119262
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,6144,0.0113045334815979
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,65536,0.10299733479817708
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,4096,0.009116799632708231
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3584,0.008683733145395915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,7168,0.013166933258374532
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,5120,0.009711999694506328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,3072,0.0077002664407094315
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2560,0.007189333438873291
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,2048,0.006433066725730896
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1536,0.006292266647020976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,1024,0.005409066875775656
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,768,0.005445333321889242
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,128,0.004434133569399515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,256,0.004717866579691568
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,65536,0.08036266962687175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,16384,0.021504000822703043
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,10240,0.013834666212399802
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,768,512,0.0048885335524876915
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,7168,0.011124266187349956
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,6144,0.00992746651172638
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,12288,0.016297599673271178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,8192,0.012365866700808208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3072,0.007085866729418437
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,5120,0.008955732981363932
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,4096,0.008071466783682505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2048,0.006135466694831848
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,3584,0.007833600044250488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1536,0.005762133498986562
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,1024,0.00517439991235733
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,512,0.004694400231043497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,2560,0.006870399912198384
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,256,0.004503466685612996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,16384,0.01944640080134074
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,65536,0.06709226767222086
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,768,0.0049898669123649595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,12288,0.01555519998073578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,512,128,0.0043605332573254905
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,10240,0.013664000233014426
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,8192,0.011804800232251484
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,7168,0.010889599720637005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,6144,0.009848533074061076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,5120,0.00897279977798462
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3584,0.007783466577529907
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2560,0.006786133348941803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,4096,0.007997866471608479
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,3072,0.007052800059318543
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,2048,0.0060928001999855045
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,768,0.0048981333772341405
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1536,0.005760000149408976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,1024,0.0050794666012128195
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,256,0.0044725333650906885
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,128,0.0042805333932240805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,65536,0.0672426700592041
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,256,512,0.004667733112970988
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,16384,0.01949653426806132
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,12288,0.01558080017566681
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,10240,0.013622400164604188
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,7168,0.010967466235160827
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,6144,0.009784533580144247
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,8192,0.011772800485293071
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3072,0.006981333096822103
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,5120,0.008906666437784832
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,4096,0.007916800181070964
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,3584,0.007809066772460937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2048,0.006053333481152853
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1536,0.0057087997595469155
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,1024,0.005053866902987162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,2560,0.0067658667763074234
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,768,0.004874666531880697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,512,0.004612266520659129
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,256,0.004419200122356415
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,2,128,128,0.004259199897448222
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,7168,0.15792105992635092
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,8192,0.17801280021667482
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,10240,0.2235637346903483
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,12288,0.2682218551635742
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,5120,0.11383786996205647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,4096,0.09542826811472574
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,6144,0.13609493573506673
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,16384,0.3541738510131836
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3584,0.08612480163574218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2048,0.05083839893341065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1536,0.044387201468149826
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,3072,0.07197653452555339
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,1024,0.0321557343006134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,2560,0.060088535149892174
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,512,0.02009920080502828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,768,0.025796266396840413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,128,0.011598933736483257
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,65536,256,0.014231466253598533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,8192,0.04904533227284749
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,10240,0.06071573495864868
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,12288,0.0723093350728353
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,16384,0.09240960280100505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,7168,0.04389760096867879
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,6144,0.0385696013768514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,5120,0.03311040004094441
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,4096,0.027875200907389326
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3584,0.025296000639597575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2560,0.019987199703852335
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,3072,0.022567466894785563
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,2048,0.01676586667696635
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1536,0.014139733711878457
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,512,0.008993066350618998
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,1024,0.011152000228563944
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,768,0.010497066378593444
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,256,0.006483200192451477
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,128,0.005525333185990652
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,10240,0.050894931952158606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,12288,0.05868586699167887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,16384,0.07977173328399659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,8192,0.04214080174763997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,16384,65536,0.3592735926310221
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,7168,0.03761920134226481
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,6144,0.03293866713841756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,4096,0.02536639968554179
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3584,0.021505065759023032
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,5120,0.02866453329722087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,3072,0.018862932920455933
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2048,0.014371200402577718
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1024,0.009432533383369445
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,1536,0.011734400192896526
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,2560,0.016739199558893837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,768,0.009469866752624512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,512,0.00734506646792094
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,128,0.004930133124192556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,256,0.005859200159708659
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,12288,0.055454933643341066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,12288,65536,0.29019091924031576
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,16384,0.07521173159281412
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,10240,0.047203199068705244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,7168,0.037251198291778566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,6144,0.030987733602523805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,5120,0.026807467142740887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,8192,0.040194133917490646
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,4096,0.022488532463709514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3584,0.01999573310216268
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,3072,0.017705599466959633
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2560,0.015611732999483744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,2048,0.013396267096201578
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1024,0.009100799759229023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,512,0.0071946665644645694
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,256,0.005677866439024607
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,1536,0.010965333382288615
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,768,0.008926933010419209
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,128,0.004775466521581014
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,10240,65536,0.25582292874654133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,12288,0.04466880162556966
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,10240,0.0409173329671224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,8192,0.032953600088755294
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,16384,0.062225067615509035
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,7168,0.029080533981323244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,6144,0.025256532430648803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,5120,0.021724800268809
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,4096,0.018402133385340372
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3584,0.017203199863433837
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2560,0.01360640029112498
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,3072,0.014919466773668923
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1024,0.008636800448099773
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,2048,0.011854933698972066
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,1536,0.010526933272679647
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,512,0.007160533467928569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,768,0.008309333523114523
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,256,0.005804799993832906
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,65536,0.20979839960734048
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,8192,128,0.0050016000866889955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,12288,0.04851626555124919
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,16384,0.07124266624450684
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,10240,0.044308265050252274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,8192,0.038227200508117676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,7168,0.032874667644500734
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,6144,0.029714133342107135
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,5120,0.024590933322906496
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3584,0.018017067511876424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,3072,0.016611199577649435
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2560,0.014550399780273438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,4096,0.020616533358891805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,2048,0.012337066729863485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1536,0.010260267059008281
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,65536,0.23060906728108727
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,768,0.007833600044250488
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,256,0.0055189331372578945
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,1024,0.007930666704972585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,128,0.0046528001626332605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,7168,512,0.006900266806284587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,10240,0.041698133945465087
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,8192,0.035835735003153485
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,16384,0.0694485346476237
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,12288,0.048002131779988605
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,5120,0.02426240046819051
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,7168,0.03156266609827678
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,6144,0.02839573423067729
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,4096,0.019689599672953285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3584,0.01830186645189921
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,3072,0.01624853312969208
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2560,0.014144000411033631
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1536,0.010141866405804952
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,1024,0.0076682666937510175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,768,0.007747200131416321
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,65536,0.22793280283610023
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,2048,0.01202453374862671
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,256,0.005453866720199585
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,128,0.0046410664916038515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,6144,512,0.0069472000002861025
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,8192,0.0368127981821696
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,10240,0.040098134676615396
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,12288,0.04558826684951782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,6144,0.027565866708755493
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,5120,0.02184106707572937
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,16384,0.06492586533228556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3584,0.01739306648572286
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,3072,0.015875200430552162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,7168,0.03258133331934611
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2560,0.014019200205802917
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,4096,0.0195743997891744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,65536,0.21838720639546713
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1024,0.007512533167997997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,2048,0.0118367999792099
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,1536,0.009866666793823243
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,256,0.0053962667783101406
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,768,0.007683200140794118
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,512,0.006750933329264323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,5120,128,0.004507733384768168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,16384,0.05252159833908081
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,12288,0.03705706596374512
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,10240,0.034363734722137454
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,7168,0.02355413238207499
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,6144,0.021452800432840983
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,8192,0.02667306661605835
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,5120,0.019052799542744955
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3584,0.014525866508483887
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,4096,0.015610667069753012
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,3072,0.012968533237775168
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,65536,0.18911147117614746
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2048,0.0102101335922877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1536,0.008710400263468424
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,1024,0.007222400108973186
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,2560,0.011803733309110005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,768,0.007803733150164287
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,512,0.006861866513888042
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,256,0.005489066739877065
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,4096,128,0.004648533463478088
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,12288,0.03678826491038005
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,10240,0.0326474666595459
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,16384,0.05392640034357706
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,7168,0.024784000714619954
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,5120,0.01920106609662374
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,8192,0.02770026723543803
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,6144,0.02209706703821818
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,4096,0.015686399737993875
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3584,0.014824533462524414
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,3072,0.01332266628742218
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2560,0.011808000008265178
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1536,0.008424533406893413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,2048,0.009966933727264404
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,1024,0.007109333574771881
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,65536,0.17675093015034993
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,768,0.007276799778143566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,512,0.006551466882228851
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,128,0.004635733366012573
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3584,256,0.005300266544024149
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,10240,0.03745813369750976
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,12288,0.045875199635823566
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,16384,0.06057920058568319
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,7168,0.029250133037567138
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,8192,0.033156265815099076
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,6144,0.02678080002466838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,5120,0.02189226746559143
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,4096,0.018311466773351034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,65536,0.20369812647501626
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2560,0.013212800025939941
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3072,0.015121066570281982
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,2048,0.010681600371996561
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1536,0.008878933389981587
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,3584,0.016667733589808144
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,1024,0.007161599894364675
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,768,0.007530666887760162
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,512,0.006621866424878438
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,128,0.004487466812133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,12288,0.039775999387105306
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,3072,256,0.005278933544953664
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,10240,0.037115732828776046
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,16384,0.05736533403396606
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,6144,0.023502933979034423
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,8192,0.029864533742268877
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,7168,0.02735146681467692
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,5120,0.020871466398239134
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,4096,0.01651413341363271
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3584,0.014946132898330688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2560,0.011539199948310852
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,65536,0.19038079579671224
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1536,0.008306133250395458
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,3072,0.01330880026022593
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,1024,0.007039999961853028
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,512,0.006230400005976359
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,2048,0.009388800462086995
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,768,0.006854400038719177
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,128,0.004492799937725067
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,12288,0.035179734230041504
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2560,256,0.005067733426888784
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,16384,0.04862080017725627
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,10240,0.03170666694641113
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,8192,0.026174932718276978
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,7168,0.02339199980099996
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,65536,0.16901013056437175
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3072,0.011678933103879293
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,6144,0.020156800746917725
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,4096,0.014729600151379904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,5120,0.017799466848373413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,3584,0.013455999890963235
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2560,0.010381866494814556
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1536,0.00834986666838328
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,768,0.007585066556930542
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,512,0.006649599969387054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,2048,0.009155199925104777
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,256,0.005403733253479004
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,128,0.004577066500981649
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,2048,1024,0.00697386662165324
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,16384,0.04578026533126831
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,12288,0.03269866704940796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,10240,0.02932159900665283
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,8192,0.025483733415603636
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,7168,0.02246826688448588
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,65536,0.1597823937733968
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,6144,0.019689599672953285
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,5120,0.017663999398549398
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,4096,0.013121066490809121
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3584,0.012246400117874146
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2560,0.009671466549237569
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,3072,0.01104213297367096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,2048,0.008610133330027263
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1024,0.0065087998906771345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,768,0.006682666639486949
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,512,0.005972266693909963
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,1536,0.0077567999561627705
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,256,0.005003733436266581
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,12288,0.0269375999768575
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,10240,0.026730666557947796
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1536,128,0.004530133306980133
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,65536,0.11978453000386555
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,8192,0.020838399728139244
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,16384,0.03639359871546428
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,7168,0.018602667252222697
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,6144,0.015869866808255514
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,5120,0.013191466530164083
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,4096,0.011060266693433126
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3584,0.009989333152770997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2048,0.007310933371384938
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,2560,0.008211199939250947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1024,0.00584853341182073
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,3072,0.009360000491142273
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,768,0.006019199887911478
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,512,0.005434666574001312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,1536,0.006938666601975759
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,128,0.00444160004456838
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,16384,0.03086613416671753
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,65536,0.10388906796773274
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,1024,256,0.00481386681397756
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,12288,0.021625600258509316
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,10240,0.02213333249092102
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,8192,0.01530346671740214
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,7168,0.013438933094342551
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,6144,0.012185600399971009
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,5120,0.010215466221173603
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,4096,0.00909546713034312
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3584,0.008404266834259034
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,3072,0.007649066547552745
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2560,0.007073066631952922
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1536,0.0061706667145093284
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,1024,0.005426133175690969
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,768,0.0055061335364977515
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,2048,0.006374399860699971
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,512,0.0051818668842315676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,128,0.004429866870244345
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,10240,0.01563093364238739
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,12288,0.01604373355706533
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,768,256,0.004666666686534882
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,8192,0.012007466952006022
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,16384,0.020890667041142782
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,65536,0.07968213558197021
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,7168,0.010850133498509724
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,6144,0.009965866804122925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,4096,0.008025600016117096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3584,0.007889066636562348
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,3072,0.006995200117429097
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2560,0.00681279997030894
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,5120,0.008982400099436443
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,2048,0.006061866879463196
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1536,0.005721599857012431
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,512,0.004759466648101807
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,256,0.004487466812133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,128,0.004369066655635833
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,1024,0.005099733173847198
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,512,768,0.0049781332413355505
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,16384,0.019429334004720054
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,65536,0.06693013509114584
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,10240,0.013682132959365845
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,12288,0.015543466806411744
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,6144,0.009902933239936828
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,7168,0.010910933216412861
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,5120,0.00895146628220876
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,8192,0.011748266220092774
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,4096,0.008025600016117096
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3584,0.007797333101431529
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,3072,0.007048533360163371
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2048,0.006040533383687338
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1536,0.005724800129731497
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,1024,0.005054933329423269
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,768,0.0048650667071342465
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,2560,0.006663466493288676
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,512,0.004599466423193614
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,256,0.004487466812133789
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,16384,0.01937920053799947
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,10240,0.013668266932169595
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,256,128,0.004308266441027323
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,65536,0.06722773710886637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,7168,0.010838400324185688
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,8192,0.01171946624914805
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,12288,0.015499732891718545
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,4096,0.008014933268229166
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,5120,0.008956799904505413
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3584,0.007717333237330119
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,3072,0.007014399766921997
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,6144,0.009859200318654377
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1536,0.005703466633955637
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,1024,0.005026133358478546
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,768,0.004856533308823904
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2560,0.006723199784755707
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,512,0.0045855998992919925
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,2048,0.006037333110968272
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,128,0.004271999994913737
TRTLLM,0.20.0,NVIDIA H100 80GB HBM3,gemm,torch_flow,fp8_block,1,128,256,0.004419200122356415
