[Perf][Qwen4Exp] Add SM100 low-latency decode GEMM plans (#59214)
Signed-off-by: namgyu-youn <namgyu.dev@gmail.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Thien Tran
parent
2eaa3bc5ac
commit
4ed438b79c
@@ -144,6 +144,73 @@ QWEN4_EXP_SM90_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] =
|
||||
}
|
||||
|
||||
|
||||
# B200 plans; the winning configs differ from the B300 table.
|
||||
QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
||||
# GDN fused B/A projection, TP=4.
|
||||
(24, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
||||
},
|
||||
# Shared-expert fused gate/up projection, TP=4.
|
||||
(320, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 160, 1, k_unroll=2, static_k=2560),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
||||
},
|
||||
# Final HC down projection, replicated in a TP=4 deployment.
|
||||
(320, 10240): {
|
||||
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240),
|
||||
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240),
|
||||
4: SkinnyGemmConfig(4, 256, 1, k_unroll=2),
|
||||
8: SkinnyGemmConfig(8, 256, 1),
|
||||
},
|
||||
# HC merged down/injection projection, replicated in a TP=4 deployment.
|
||||
(336, 10240): {
|
||||
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240),
|
||||
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240),
|
||||
8: SkinnyGemmConfig(8, 256, 3, static_k=10240),
|
||||
},
|
||||
# QSA indexer Q/K projection, replicated in a TP=4 deployment.
|
||||
(640, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||
},
|
||||
# GDN and QSA output projections, TP=4.
|
||||
(2560, 1536): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=1536),
|
||||
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=1536),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=1536),
|
||||
},
|
||||
# QSA fused QKV/gate projection, TP=4.
|
||||
(3584, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||
},
|
||||
# GDN fused QKVZ projection, TP=4.
|
||||
(4096, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
||||
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||
},
|
||||
# LM head, TP=4.
|
||||
(62080, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _is_sm100() -> bool:
|
||||
return current_platform.is_device_capability((10, 0))
|
||||
|
||||
|
||||
def _is_sm103() -> bool:
|
||||
return current_platform.is_device_capability((10, 3))
|
||||
|
||||
@@ -155,6 +222,8 @@ def _is_sm90() -> bool:
|
||||
def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]:
|
||||
if _is_sm103():
|
||||
return QWEN4_EXP_GEMM_PLANS
|
||||
if _is_sm100():
|
||||
return QWEN4_EXP_SM100_GEMM_PLANS
|
||||
if _is_sm90():
|
||||
return QWEN4_EXP_SM90_GEMM_PLANS
|
||||
return {}
|
||||
|
||||
Reference in New Issue
Block a user