From 9ef764fe038f0f1fde2a48338c1a15d5fe965c5b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Aug 2026 10:47:49 +0000 Subject: [PATCH 1/5] [script] Add H100 config and scripts --- ...12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 8 + ...12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 7 + ...24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json | 7 + ...,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json | 38 +++ ...,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json | 38 +++ ...LEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json | 7 + ...LEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json | 7 + ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 50 ++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 50 ++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 62 +++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 50 ++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 50 ++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 98 +++++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 74 +++++ ...torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json | 74 +++++ test/benchmark/bench_agents.py | 263 ++++++++++++++++++ test/start_scripts/qwen35/qwen35_pd_2p1d.sh | 40 ++- 32 files changed, 1935 insertions(+), 8 deletions(-) create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=12,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=24,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=12,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=24,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=12,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=24,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/gated_rmsnorm_forward:v1/{N=128,has_bias=false,weight_dtype=torch.bfloat16,x_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=1,Q_HEAD_NUM=6,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=2,Q_HEAD_NUM=12,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=10240,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=1536,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=3072,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=4352,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=17408,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=3584,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=4120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=7168,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8240,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8704,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=8704,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=4352,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=8704,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json create mode 100644 test/benchmark/bench_agents.py diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..f55b637832 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "4": { + "BK": 128, + "BV": 128, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..b6e5109b62 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=16,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "8": { + "BK": 128, + "BV": 64, + "num_stages": 4, + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..fc9fc9a4ae --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "4": { + "BK": 128, + "BV": 64, + "num_stages": 3, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..b6e5109b62 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=32,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "8": { + "BK": 128, + "BV": 64, + "num_stages": 4, + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..fc9fc9a4ae --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "4": { + "BK": 128, + "BV": 64, + "num_stages": 3, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..f1159e4357 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_fwd_o/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,8 @@ +{ + "8": { + "BK": 64, + "BV": 128, + "num_stages": 3, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..d831f32c4a --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=12,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,7 @@ +{ + "4": { + "BV": 32, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..2af1b86e90 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=24,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,7 @@ +{ + "8": { + "BV": 32, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=12,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=12,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..613295d0a2 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=12,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,38 @@ +{ + "1": { + "num_warps": 1 + }, + "1024": { + "num_warps": 8 + }, + "128": { + "num_warps": 8 + }, + "16": { + "num_warps": 8 + }, + "2048": { + "num_warps": 1 + }, + "256": { + "num_warps": 1 + }, + "32": { + "num_warps": 8 + }, + "4": { + "num_warps": 8 + }, + "4096": { + "num_warps": 1 + }, + "64": { + "num_warps": 1 + }, + "8": { + "num_warps": 1 + }, + "8192": { + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=24,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=24,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..3b52281b64 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_local_cumsum_scalar/{B=1,BT=64,H=24,IS_VARLEN=true,REVERSE=false}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,38 @@ +{ + "1": { + "num_warps": 2 + }, + "1024": { + "num_warps": 2 + }, + "128": { + "num_warps": 2 + }, + "16": { + "num_warps": 2 + }, + "2048": { + "num_warps": 4 + }, + "256": { + "num_warps": 2 + }, + "32": { + "num_warps": 2 + }, + "4": { + "num_warps": 8 + }, + "4096": { + "num_warps": 2 + }, + "64": { + "num_warps": 2 + }, + "8": { + "num_warps": 2 + }, + "8192": { + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=12,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=12,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..2202ff233e --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=12,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,7 @@ +{ + "4": { + "BK": 64, + "num_stages": 4, + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=24,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=24,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..38586cbbd2 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/chunk_scaled_dot_kkt_fwd/{BT=64,H=24,IS_VARLEN=true,K=128}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,7 @@ +{ + "8": { + "BK": 64, + "num_stages": 3, + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=12,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=12,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..1f9c2bf0ad --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=12,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,50 @@ +{ + "1": { + "BLK_HEADS": 16, + "num_warps": 1 + }, + "1024": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "128": { + "BLK_HEADS": 16, + "num_warps": 1 + }, + "16": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "2048": { + "BLK_HEADS": 16, + "num_warps": 1 + }, + "256": { + "BLK_HEADS": 16, + "num_warps": 1 + }, + "32": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "4": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "4096": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "64": { + "BLK_HEADS": 16, + "num_warps": 2 + }, + "8": { + "BLK_HEADS": 64, + "num_warps": 4 + }, + "8192": { + "BLK_HEADS": 16, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=24,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=24,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..65fb4da7fe --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/fused_gdn_gating:v1/{NUM_HEADS=24,a_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,50 @@ +{ + "1": { + "BLK_HEADS": 64, + "num_warps": 1 + }, + "1024": { + "BLK_HEADS": 32, + "num_warps": 4 + }, + "128": { + "BLK_HEADS": 64, + "num_warps": 2 + }, + "16": { + "BLK_HEADS": 4, + "num_warps": 1 + }, + "2048": { + "BLK_HEADS": 64, + "num_warps": 2 + }, + "256": { + "BLK_HEADS": 32, + "num_warps": 1 + }, + "32": { + "BLK_HEADS": 32, + "num_warps": 4 + }, + "4": { + "BLK_HEADS": 32, + "num_warps": 1 + }, + "4096": { + "BLK_HEADS": 64, + "num_warps": 2 + }, + "64": { + "BLK_HEADS": 64, + "num_warps": 1 + }, + "8": { + "BLK_HEADS": 8, + "num_warps": 1 + }, + "8192": { + "BLK_HEADS": 32, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/gated_rmsnorm_forward:v1/{N=128,has_bias=false,weight_dtype=torch.bfloat16,x_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/gated_rmsnorm_forward:v1/{N=128,has_bias=false,weight_dtype=torch.bfloat16,x_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..b890b65c4b --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/gated_rmsnorm_forward:v1/{N=128,has_bias=false,weight_dtype=torch.bfloat16,x_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,62 @@ +{ + "12": { + "BLOCK_N": 256, + "num_warps": 2 + }, + "12288": { + "BLOCK_N": 64, + "num_warps": 1 + }, + "1536": { + "BLOCK_N": 64, + "num_warps": 1 + }, + "192": { + "BLOCK_N": 256, + "num_warps": 2 + }, + "196608": { + "BLOCK_N": 64, + "num_warps": 1 + }, + "24": { + "BLOCK_N": 256, + "num_warps": 2 + }, + "24576": { + "BLOCK_N": 128, + "num_warps": 1 + }, + "3072": { + "BLOCK_N": 128, + "num_warps": 1 + }, + "384": { + "BLOCK_N": 512, + "num_warps": 1 + }, + "48": { + "BLOCK_N": 512, + "num_warps": 1 + }, + "49152": { + "BLOCK_N": 128, + "num_warps": 1 + }, + "6144": { + "BLOCK_N": 128, + "num_warps": 1 + }, + "768": { + "BLOCK_N": 512, + "num_warps": 1 + }, + "96": { + "BLOCK_N": 512, + "num_warps": 8 + }, + "98304": { + "BLOCK_N": 128, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=1,Q_HEAD_NUM=6,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=1,Q_HEAD_NUM=6,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..519b594e27 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=1,Q_HEAD_NUM=6,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,50 @@ +{ + "1": { + "num_stages": 2, + "num_warps": 1 + }, + "1024": { + "num_stages": 3, + "num_warps": 1 + }, + "128": { + "num_stages": 3, + "num_warps": 2 + }, + "16": { + "num_stages": 2, + "num_warps": 1 + }, + "2048": { + "num_stages": 5, + "num_warps": 2 + }, + "256": { + "num_stages": 3, + "num_warps": 1 + }, + "32": { + "num_stages": 5, + "num_warps": 2 + }, + "4": { + "num_stages": 5, + "num_warps": 2 + }, + "4096": { + "num_stages": 1, + "num_warps": 2 + }, + "64": { + "num_stages": 5, + "num_warps": 2 + }, + "8": { + "num_stages": 5, + "num_warps": 8 + }, + "8192": { + "num_stages": 5, + "num_warps": 2 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=2,Q_HEAD_NUM=12,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=2,Q_HEAD_NUM=12,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..b8655ab8f0 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/mrope_triton_fused:v1/{HEAD_DIM=256,K_HEAD_NUM=2,Q_HEAD_NUM=12,dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,50 @@ +{ + "1": { + "num_stages": 5, + "num_warps": 8 + }, + "1024": { + "num_stages": 3, + "num_warps": 4 + }, + "128": { + "num_stages": 3, + "num_warps": 4 + }, + "16": { + "num_stages": 3, + "num_warps": 4 + }, + "2048": { + "num_stages": 3, + "num_warps": 1 + }, + "256": { + "num_stages": 3, + "num_warps": 1 + }, + "32": { + "num_stages": 2, + "num_warps": 1 + }, + "4": { + "num_stages": 3, + "num_warps": 1 + }, + "4096": { + "num_stages": 3, + "num_warps": 1 + }, + "64": { + "num_stages": 2, + "num_warps": 1 + }, + "8": { + "num_stages": 3, + "num_warps": 2 + }, + "8192": { + "num_stages": 1, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=10240,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=10240,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..8e6ce03396 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=10240,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=1536,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=1536,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..03fee9bc5c --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=1536,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=3072,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=3072,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..d7d47c6ac4 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=3072,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=4352,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=4352,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..fd93293899 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=4352,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 8 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=17408,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=17408,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..4bcad9ae59 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=17408,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 64, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=3584,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=3584,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..7dad4079dc --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=3584,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=4120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=4120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..93828ebc22 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=4120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 2, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=7168,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=7168,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..5d194aec64 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=7168,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 2 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 6, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8240,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8240,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..bd3f906e29 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8240,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 2 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 8 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8704,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8704,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..93bb635a33 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=5120,N=8704,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 3, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=8704,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=8704,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..21c8120599 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/scaled_mm_per_token:v1/{K=8704,N=5120,b_scale_kind=tensor,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,98 @@ +{ + "1": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "1024": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "128": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 128, + "GROUP_M": 8, + "num_stages": 4, + "num_warps": 4 + }, + "16": { + "BLOCK_K": 256, + "BLOCK_M": 16, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "2048": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "256": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "32": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 8 + }, + "4": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "4096": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "64": { + "BLOCK_K": 256, + "BLOCK_M": 64, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8": { + "BLOCK_K": 256, + "BLOCK_M": 8, + "BLOCK_N": 64, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + }, + "8192": { + "BLOCK_K": 128, + "BLOCK_M": 64, + "BLOCK_N": 256, + "GROUP_M": 8, + "num_stages": 5, + "num_warps": 4 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=4352,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=4352,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..2fd4af0080 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=4352,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,74 @@ +{ + "1": { + "BLOCK_M": 128, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 8 + }, + "1024": { + "BLOCK_M": 8, + "BLOCK_N": 256, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "128": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "16": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "2048": { + "BLOCK_M": 32, + "BLOCK_N": 128, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "256": { + "BLOCK_M": 8, + "BLOCK_N": 256, + "NUM_STAGES": 2, + "num_warps": 8 + }, + "32": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "4": { + "BLOCK_M": 1, + "BLOCK_N": 64, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "4096": { + "BLOCK_M": 32, + "BLOCK_N": 256, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "64": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "8": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 4, + "num_warps": 8 + }, + "8192": { + "BLOCK_M": 8, + "BLOCK_N": 128, + "NUM_STAGES": 4, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=8704,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=8704,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 0000000000..07633c2c92 --- /dev/null +++ b/lightllm/common/triton_utils/autotune_kernel_configs/triton_3.6.0/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=8704,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,74 @@ +{ + "1": { + "BLOCK_M": 32, + "BLOCK_N": 256, + "NUM_STAGES": 2, + "num_warps": 8 + }, + "1024": { + "BLOCK_M": 32, + "BLOCK_N": 128, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "128": { + "BLOCK_M": 8, + "BLOCK_N": 128, + "NUM_STAGES": 2, + "num_warps": 4 + }, + "16": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "2048": { + "BLOCK_M": 32, + "BLOCK_N": 256, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "256": { + "BLOCK_M": 8, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 4 + }, + "32": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 2, + "num_warps": 1 + }, + "4": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 2, + "num_warps": 1 + }, + "4096": { + "BLOCK_M": 8, + "BLOCK_N": 128, + "NUM_STAGES": 4, + "num_warps": 1 + }, + "64": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 1, + "num_warps": 1 + }, + "8": { + "BLOCK_M": 1, + "BLOCK_N": 256, + "NUM_STAGES": 4, + "num_warps": 4 + }, + "8192": { + "BLOCK_M": 8, + "BLOCK_N": 128, + "NUM_STAGES": 4, + "num_warps": 1 + } +} \ No newline at end of file diff --git a/test/benchmark/bench_agents.py b/test/benchmark/bench_agents.py new file mode 100644 index 0000000000..882428b821 --- /dev/null +++ b/test/benchmark/bench_agents.py @@ -0,0 +1,263 @@ +import argparse +import asyncio +import math +import time +from typing import Any, Dict, List, Optional + +from openai import AsyncOpenAI + +# Agent-style open-loop benchmark. +# - N sessions, each with a distinct long system prompt (so cache_aware spreads +# them across P nodes); turns within a session share that prefix (KV reuse). +# - Fixed turn-emission rate (open loop): the scheduler fires one turn every +# 1/QPS seconds. If every session is still waiting on its previous turn, the +# tick is counted as blocked -> the saturation signal. +# - Prefix grows each turn (full history re-sent), mirroring real multi-turn agents. + + +def percentile(data: List[float], p: float) -> float: + if not data: + return 0.0 + s = sorted(data) + idx = (p / 100.0) * (len(s) - 1) + lo, hi = math.floor(idx), math.ceil(idx) + if lo == hi: + return s[int(idx)] + w = idx - lo + return s[lo] * (1 - w) + s[hi] * w + + +def stats(values: List[float]) -> Dict[str, float]: + if not values: + return {k: 0.0 for k in ["mean", "p50", "p90", "p95", "p99", "max"]} + return { + "mean": sum(values) / len(values), + "p50": percentile(values, 50), + "p90": percentile(values, 90), + "p95": percentile(values, 95), + "p99": percentile(values, 99), + "max": max(values), + } + + +def filler(seed: int, target_chars: int) -> str: + """Deterministic per-seed text of ~target_chars. Distinct seed -> distinct bytes.""" + out = [] + n = 0 + k = 0 + while n < target_chars: + word = f"{seed:08d}-agentctx-{k} " + out.append(word) + n += len(word) + k += 1 + return "".join(out)[:target_chars] + + +def make_system_prompt(seed: int, isl: int) -> str: + # ~4 chars/token; distinct per session so cache_aware hashes the prefix to + # different P nodes across sessions. + return filler(seed, isl * 4) + + +def make_user_turn(session_seed: int, turn_idx: int) -> str: + return f"[session {session_seed} turn {turn_idx}] " + filler(session_seed * 31 + turn_idx, 256) + + +class Session: + __slots__ = ("sid", "system", "history", "pending", "turns_done", "birth") + + def __init__(self, sid: int, system: str): + self.sid = sid + self.system = system + self.history: List[Dict[str, str]] = [] + self.pending = False + self.turns_done = 0 + self.birth = 0.0 + + def reset(self, system: str): + self.system = system + self.history = [] + self.pending = False + self.turns_done = 0 + + +async def do_turn( + client: AsyncOpenAI, + model: str, + sess: Session, + osl: int, + metrics: List[Dict[str, Any]], + warmup_until: float, + rr_idx: List[int], +): + user_msg = make_user_turn(sess.sid, sess.turns_done) + messages = [{"role": "system", "content": sess.system}] + sess.history + [{"role": "user", "content": user_msg}] + sess.pending = True + t0 = time.perf_counter() + first_t = second_t = last_t = None + ntok = 0 + err = None + try: + resp = await client.chat.completions.create( + model=model, + messages=messages, + max_tokens=osl, + temperature=0.7, + stream=True, + ) + async for chunk in resp: + if not (chunk.choices and chunk.choices[0].delta): + continue + delta = chunk.choices[0].delta + text = getattr(delta, "content", None) or getattr(delta, "reasoning_content", None) + if text: + ntok += 1 + now = time.perf_counter() + if ntok == 1: + first_t = now + elif ntok == 2: + second_t = now + last_t = now + end = time.perf_counter() + if ntok < 1: + err = "no tokens" + else: + # append assistant reply so the next turn's prefix includes it + sess.history.append({"role": "assistant", "content": "(reply)"}) + if time.perf_counter() >= warmup_until: + metrics.append({ + "ttft": (first_t - t0) * 1000, + "itl": ((second_t - first_t) * 1000) if (first_t and second_t) else None, + "tpot": (((last_t - first_t) / (ntok - 1)) * 1000) if (ntok > 1 and first_t and last_t) else 0.0, + "lat": (end - t0) * 1000, + "turn_idx": sess.turns_done, + "sid": sess.sid, + }) + except Exception as e: # keep the run alive; record failure count only + err = str(e) + finally: + sess.turns_done += 1 + sess.pending = False + if err: + metrics.append({"error": err, "turn_idx": sess.turns_done, "sid": sess.sid}) + rr_idx[0] += 1 + + +async def run(args): + client = AsyncOpenAI(api_key=args.api_key, base_url=args.base_url) + interval = 1.0 / args.qps if args.qps > 0 else 0.0 + + # fresh set of sessions; distinct system prompt per session + sessions = [Session(sid=sid, system=make_system_prompt(sid, args.isl)) for sid in range(args.num_sessions)] + rr_idx = [0] # round-robin cursor + metrics: List[Dict[str, Any]] = [] + fired = 0 + blocked = 0 + warmup_until = time.perf_counter() + args.warmup_sec + + print( + f"agent bench: qps={args.qps} sessions={args.num_sessions} turns/session={args.turns_per_session} " + f"isl={args.isl} osl={args.osl} duration={args.duration_sec}s warmup={args.warmup_sec}s" + ) + + start = time.perf_counter() + end_at = start + args.duration_sec + next_tick = start + + async def fire_one(): + nonlocal fired, blocked + # pick next non-pending session (round-robin over ready ones) + n = len(sessions) + chosen: Optional[Session] = None + for i in range(n): + s = sessions[(rr_idx[0] + i) % n] + if not s.pending: + chosen = s + rr_idx[0] = (rr_idx[0] + i + 1) % n + break + if chosen is None: + blocked += 1 # saturation: no session has finished its previous turn + return + fired += 1 + # retire a finished session into a fresh identity so the pool stays populated + if chosen.turns_done >= args.turns_per_session: + chosen.reset(system=make_system_prompt(seed=10_000_000 + rr_idx[0] + fired, isl=args.isl)) + chosen.birth = time.perf_counter() + asyncio.create_task(do_turn(client, args.model, chosen, args.osl, metrics, warmup_until, rr_idx)) + + # open-loop arrival loop: fire at fixed rate regardless of completions + while True: + now = time.perf_counter() + if now >= end_at: + break + if now >= next_tick: + await fire_one() + next_tick += interval + if interval == 0: + next_tick = now + else: + await asyncio.sleep(min(next_tick - now, 0.01)) + + # drain in-flight + deadline = time.perf_counter() + args.drain_sec + while any(s.pending for s in sessions) and time.perf_counter() < deadline: + await asyncio.sleep(0.05) + + total_time = time.perf_counter() - start + summarize(metrics, fired, blocked, total_time, args) + + +def summarize(metrics, fired, blocked, total_time, args): + ok = [m for m in metrics if "error" not in m] + fail = [m for m in metrics if "error" in m] + ttfts = [m["ttft"] for m in ok] + tpots = [m["tpot"] for m in ok if m["tpot"] > 0] + lats = [m["lat"] for m in ok] + + # break TTFT down by turn index to show prefix-cache benefit (turn 0 vs later) + by_turn: Dict[int, List[float]] = {} + for m in ok: + by_turn.setdefault(m["turn_idx"], []).append(m["ttft"]) + + def line(name, s): + print(f" {name:<30} mean={s['mean']:8.1f} p50={s['p50']:8.1f} " + f"p90={s['p90']:8.1f} p95={s['p95']:8.1f} p99={s['p99']:8.1f} max={s['max']:8.1f} (ms)") + + print("\n" + "=" * 96) + print("AGENT OPEN-LOOP BENCHMARK RESULTS") + print("=" * 96) + print(f" target QPS : {args.qps}") + print(f" fired turns : {fired} blocked ticks (saturation): {blocked}") + print(f" completed (ok/fail): {len(ok)} / {len(fail)}") + print(f" achieved turn QPS : {len(ok) / total_time:.2f} /s (wall {total_time:.1f}s)") + print("-" * 96) + line("TTFT", stats(ttfts)) + line("TPOT", stats(tpots)) + line("Total turn latency", stats(lats)) + print("-" * 96) + print(" TTFT by turn index (prefix grows each turn; later turns should be faster if cache hits):") + for idx in sorted(by_turn): + s = stats(by_turn[idx]) + print(f" turn {idx:<3} n={len(by_turn[idx]):<5} mean={s['mean']:8.1f} p50={s['p50']:8.1f} p90={s['p90']:8.1f} (ms)") + print("=" * 96 + "\n") + + +def main(): + p = argparse.ArgumentParser(description="Agent multi-turn open-loop benchmark") + p.add_argument("--api-key", default="EMPTY") + p.add_argument("--base-url", default="http://127.0.0.1:16666/v1") + p.add_argument("--model", default="qwen35") + p.add_argument("--qps", type=float, default=5.0, help="turn emission rate (open loop)") + p.add_argument("--num-sessions", type=int, default=32, help="concurrent sessions; distinct prefix each") + p.add_argument("--turns-per-session", type=int, default=6, help="turns before a session is recycled") + p.add_argument("--isl", type=int, default=2048, help="system-prompt length (tokens) per session") + p.add_argument("--osl", type=int, default=256, help="max output tokens per turn") + p.add_argument("--duration-sec", type=float, default=60.0, help="emission window") + p.add_argument("--warmup-sec", type=float, default=10.0, help="discard metrics before this") + p.add_argument("--drain-sec", type=float, default=120.0, help="max wait for in-flight turns after window") + asyncio.run(run(p.parse_args())) + + +if __name__ == "__main__": + main() + diff --git a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh index 51b8d7aafa..e53953ac0c 100755 --- a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh +++ b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh @@ -14,9 +14,9 @@ if [[ -n "${4:-}" ]]; then CHAT_TEMPLATE_ARGS=(--chat_template "$4") fi +# export PYTORCH_ALLOC_CONF=expandable_segments:True export LOADWORKER=8 export LIGHTLLM_TRITON_AUTOTUNE_LEVEL=1 -export LIGHTLLM_FP8_GEMM=sgl export LIGHTLLM_ANTHROPIC_ENABLE_PDF_PARSING=1 export LIGHTLLM_LOG_LEVEL=debug @@ -24,7 +24,31 @@ export LIGHTLLM_LOG_LEVEL=debug export NO_PROXY="${NO_PROXY:+${NO_PROXY},}127.0.0.1,localhost" export no_proxy="${NO_PROXY}" -COMMON_ARGS=( +P_COMMON_ARGS=( + --model_dir "${MODEL_DIR}" + --model_name qwen35_27b + --graph_max_batch_size 8 + --running_max_req_size 8 + --mem_fraction 0.80 + --max_image_token_count 4096 + --max_image_pixels 3686400 + --batch_max_tokens 8192 + --linear_att_cache_size 500 + --linear_att_hash_page_size 4096 + --linear_att_page_block_num 8 + --quant_type fp8w8a8-pt-sgl + --mtp_mode eagle_with_att + --mtp_draft_model_dir "${MODEL_DIR}" + --mtp_step 3 + "${CHAT_TEMPLATE_ARGS[@]}" + --pd_trans_mode nixl + --pd_kv_page_size 4096 + --pd_master_ip 127.0.0.1 + --pd_master_port "${PORT}" + --enable_prefill_cudagraph +) + +D_COMMON_ARGS=( --model_dir "${MODEL_DIR}" --model_name qwen35_27b --graph_max_batch_size 64 @@ -32,16 +56,16 @@ COMMON_ARGS=( --mem_fraction 0.80 --max_image_token_count 4096 --max_image_pixels 3686400 - --batch_max_tokens 8192 + --batch_max_tokens 256 --linear_att_cache_size 500 --linear_att_hash_page_size 4096 --linear_att_page_block_num 8 - --quant_type fp8w8a8-pt-triton + --quant_type fp8w8a8-pt-sgl --mtp_mode eagle_with_att --mtp_draft_model_dir "${MODEL_DIR}" --mtp_step 3 "${CHAT_TEMPLATE_ARGS[@]}" - --pd_trans_mode nccl + --pd_trans_mode nixl --pd_kv_page_size 4096 --pd_master_ip 127.0.0.1 --pd_master_port "${PORT}" @@ -60,7 +84,7 @@ trap 'exit 143' TERM # Prefill 1: TP2 on GPUs 0-1. CUDA_VISIBLE_DEVICES=0,1 python -m lightllm.server.api_server \ - "${COMMON_ARGS[@]}" \ + "${P_COMMON_ARGS[@]}" \ --run_mode prefill \ --enable_cpu_cache \ --cpu_cache_storage_size "${CPU_CACHE_SIZE}" \ @@ -73,7 +97,7 @@ PIDS+=("$!") # Prefill 2: TP2 on GPUs 2-3. CUDA_VISIBLE_DEVICES=2,3 python -m lightllm.server.api_server \ - "${COMMON_ARGS[@]}" \ + "${P_COMMON_ARGS[@]}" \ --run_mode prefill \ --enable_cpu_cache \ --cpu_cache_storage_size "${CPU_CACHE_SIZE}" \ @@ -86,7 +110,7 @@ PIDS+=("$!") # Decode: TP4 on GPUs 4-7. CUDA_VISIBLE_DEVICES=4,5,6,7 python -m lightllm.server.api_server \ - "${COMMON_ARGS[@]}" \ + "${D_COMMON_ARGS[@]}" \ --run_mode decode \ --tp 4 \ --host 0.0.0.0 \ From da6e55d481f5e6515ac8aa38c50f1d6d932c0ba4 Mon Sep 17 00:00:00 2001 From: baishihao Date: Tue, 4 Aug 2026 19:11:43 +0800 Subject: [PATCH 2/5] fix benchmark script formatting --- test/benchmark/bench_agents.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/test/benchmark/bench_agents.py b/test/benchmark/bench_agents.py index 882428b821..91a4b50935 100644 --- a/test/benchmark/bench_agents.py +++ b/test/benchmark/bench_agents.py @@ -125,14 +125,18 @@ async def do_turn( # append assistant reply so the next turn's prefix includes it sess.history.append({"role": "assistant", "content": "(reply)"}) if time.perf_counter() >= warmup_until: - metrics.append({ - "ttft": (first_t - t0) * 1000, - "itl": ((second_t - first_t) * 1000) if (first_t and second_t) else None, - "tpot": (((last_t - first_t) / (ntok - 1)) * 1000) if (ntok > 1 and first_t and last_t) else 0.0, - "lat": (end - t0) * 1000, - "turn_idx": sess.turns_done, - "sid": sess.sid, - }) + metrics.append( + { + "ttft": (first_t - t0) * 1000, + "itl": ((second_t - first_t) * 1000) if (first_t and second_t) else None, + "tpot": (((last_t - first_t) / (ntok - 1)) * 1000) + if (ntok > 1 and first_t and last_t) + else 0.0, + "lat": (end - t0) * 1000, + "turn_idx": sess.turns_done, + "sid": sess.sid, + } + ) except Exception as e: # keep the run alive; record failure count only err = str(e) finally: @@ -220,8 +224,10 @@ def summarize(metrics, fired, blocked, total_time, args): by_turn.setdefault(m["turn_idx"], []).append(m["ttft"]) def line(name, s): - print(f" {name:<30} mean={s['mean']:8.1f} p50={s['p50']:8.1f} " - f"p90={s['p90']:8.1f} p95={s['p95']:8.1f} p99={s['p99']:8.1f} max={s['max']:8.1f} (ms)") + print( + f" {name:<30} mean={s['mean']:8.1f} p50={s['p50']:8.1f} " + f"p90={s['p90']:8.1f} p95={s['p95']:8.1f} p99={s['p99']:8.1f} max={s['max']:8.1f} (ms)" + ) print("\n" + "=" * 96) print("AGENT OPEN-LOOP BENCHMARK RESULTS") @@ -238,7 +244,10 @@ def line(name, s): print(" TTFT by turn index (prefix grows each turn; later turns should be faster if cache hits):") for idx in sorted(by_turn): s = stats(by_turn[idx]) - print(f" turn {idx:<3} n={len(by_turn[idx]):<5} mean={s['mean']:8.1f} p50={s['p50']:8.1f} p90={s['p90']:8.1f} (ms)") + print( + f" turn {idx:<3} n={len(by_turn[idx]):<5} mean={s['mean']:8.1f} " + f"p50={s['p50']:8.1f} p90={s['p90']:8.1f} (ms)" + ) print("=" * 96 + "\n") @@ -260,4 +269,3 @@ def main(): if __name__ == "__main__": main() - From 4da99ab37e1b802d7e984c47fd49ef178aeb3307 Mon Sep 17 00:00:00 2001 From: baishihao Date: Tue, 4 Aug 2026 19:19:24 +0800 Subject: [PATCH 3/5] document agent benchmark usage --- test/benchmark/bench_agents.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/benchmark/bench_agents.py b/test/benchmark/bench_agents.py index 91a4b50935..93458d7ad2 100644 --- a/test/benchmark/bench_agents.py +++ b/test/benchmark/bench_agents.py @@ -1,3 +1,13 @@ +# Example: +# python test/benchmark/bench_agents.py \ +# --qps 5 \ +# --num-sessions 32 \ +# --turns-per-session 6 \ +# --isl 4096 \ +# --osl 256 \ +# --duration-sec 60 \ +# --warmup-sec 10 + import argparse import asyncio import math From f29594ce30d577c4d26c850c95a81f04f8baf796 Mon Sep 17 00:00:00 2001 From: baishihao Date: Tue, 4 Aug 2026 19:28:16 +0800 Subject: [PATCH 4/5] tune linear attention page size --- test/start_scripts/qwen35/qwen35_pd_2p1d.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh index e53953ac0c..55c57a8d23 100755 --- a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh +++ b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh @@ -34,7 +34,7 @@ P_COMMON_ARGS=( --max_image_pixels 3686400 --batch_max_tokens 8192 --linear_att_cache_size 500 - --linear_att_hash_page_size 4096 + --linear_att_hash_page_size 2048 --linear_att_page_block_num 8 --quant_type fp8w8a8-pt-sgl --mtp_mode eagle_with_att @@ -58,7 +58,7 @@ D_COMMON_ARGS=( --max_image_pixels 3686400 --batch_max_tokens 256 --linear_att_cache_size 500 - --linear_att_hash_page_size 4096 + --linear_att_hash_page_size 2048 --linear_att_page_block_num 8 --quant_type fp8w8a8-pt-sgl --mtp_mode eagle_with_att From 9fa7e911de3e377c52d4e8fcbffcffd536d422df Mon Sep 17 00:00:00 2001 From: baishihao Date: Tue, 4 Aug 2026 19:30:44 +0800 Subject: [PATCH 5/5] switch PD transport to NCCL --- test/start_scripts/qwen35/qwen35_pd_2p1d.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh index 55c57a8d23..f50840e5e0 100755 --- a/test/start_scripts/qwen35/qwen35_pd_2p1d.sh +++ b/test/start_scripts/qwen35/qwen35_pd_2p1d.sh @@ -41,7 +41,7 @@ P_COMMON_ARGS=( --mtp_draft_model_dir "${MODEL_DIR}" --mtp_step 3 "${CHAT_TEMPLATE_ARGS[@]}" - --pd_trans_mode nixl + --pd_trans_mode nccl --pd_kv_page_size 4096 --pd_master_ip 127.0.0.1 --pd_master_port "${PORT}" @@ -58,14 +58,12 @@ D_COMMON_ARGS=( --max_image_pixels 3686400 --batch_max_tokens 256 --linear_att_cache_size 500 - --linear_att_hash_page_size 2048 - --linear_att_page_block_num 8 --quant_type fp8w8a8-pt-sgl --mtp_mode eagle_with_att --mtp_draft_model_dir "${MODEL_DIR}" --mtp_step 3 "${CHAT_TEMPLATE_ARGS[@]}" - --pd_trans_mode nixl + --pd_trans_mode nccl --pd_kv_page_size 4096 --pd_master_ip 127.0.0.1 --pd_master_port "${PORT}"