[Perf][Qwen4Exp] Add SM100 low-latency decode GEMM plans (#59214)
Signed-off-by: namgyu-youn <namgyu.dev@gmail.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Thien Tran
parent
2eaa3bc5ac
commit
4ed438b79c
@@ -85,6 +85,12 @@ QWEN4_EXP_SM90_CASES = [
|
|||||||
for num_tokens, config in plans.items()
|
for num_tokens, config in plans.items()
|
||||||
]
|
]
|
||||||
|
|
||||||
|
QWEN4_EXP_SM100_CASES = [
|
||||||
|
(n, k, num_tokens, config)
|
||||||
|
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS.items()
|
||||||
|
for num_tokens, config in plans.items()
|
||||||
|
]
|
||||||
|
|
||||||
EXPECTED_CUTE_CONFIGS = {
|
EXPECTED_CUTE_CONFIGS = {
|
||||||
(3072, 7168, 1): (224, 3, 4, 8),
|
(3072, 7168, 1): (224, 3, 4, 8),
|
||||||
(3072, 7168, 2): (128, 3, 2, 8),
|
(3072, 7168, 2): (128, 3, 2, 8),
|
||||||
@@ -717,6 +723,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
|||||||
"capability,expected_plans",
|
"capability,expected_plans",
|
||||||
[
|
[
|
||||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
||||||
|
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
|
||||||
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
||||||
((8, 0), {}),
|
((8, 0), {}),
|
||||||
],
|
],
|
||||||
@@ -909,6 +916,29 @@ def test_qwen4_exp_sm90_selected_shapes(
|
|||||||
assert cosine > 0.999
|
assert cosine > 0.999
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM100_CASES)
|
||||||
|
def test_qwen4_exp_sm100_selected_shapes(
|
||||||
|
n: int,
|
||||||
|
k: int,
|
||||||
|
num_tokens: int,
|
||||||
|
config: SkinnyGemmConfig,
|
||||||
|
) -> None:
|
||||||
|
_require_capability_and_cute((10, 0))
|
||||||
|
torch.manual_seed(42 + num_tokens)
|
||||||
|
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||||
|
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||||
|
|
||||||
|
selected = qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS[(n, k)][num_tokens]
|
||||||
|
assert selected == config
|
||||||
|
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
|
||||||
|
|
||||||
|
reference = torch.nn.functional.linear(x, weight)
|
||||||
|
cosine = torch.nn.functional.cosine_similarity(
|
||||||
|
output.float().flatten(), reference.float().flatten(), dim=0
|
||||||
|
).item()
|
||||||
|
assert cosine > 0.999
|
||||||
|
|
||||||
|
|
||||||
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
||||||
_require_sm103_and_cute()
|
_require_sm103_and_cute()
|
||||||
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||||
|
|||||||
@@ -144,6 +144,73 @@ QWEN4_EXP_SM90_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] =
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# B200 plans; the winning configs differ from the B300 table.
|
||||||
|
QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
||||||
|
# GDN fused B/A projection, TP=4.
|
||||||
|
(24, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
||||||
|
},
|
||||||
|
# Shared-expert fused gate/up projection, TP=4.
|
||||||
|
(320, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
4: SkinnyGemmConfig(4, 160, 1, k_unroll=2, static_k=2560),
|
||||||
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
||||||
|
},
|
||||||
|
# Final HC down projection, replicated in a TP=4 deployment.
|
||||||
|
(320, 10240): {
|
||||||
|
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240),
|
||||||
|
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240),
|
||||||
|
4: SkinnyGemmConfig(4, 256, 1, k_unroll=2),
|
||||||
|
8: SkinnyGemmConfig(8, 256, 1),
|
||||||
|
},
|
||||||
|
# HC merged down/injection projection, replicated in a TP=4 deployment.
|
||||||
|
(336, 10240): {
|
||||||
|
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240),
|
||||||
|
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240),
|
||||||
|
8: SkinnyGemmConfig(8, 256, 3, static_k=10240),
|
||||||
|
},
|
||||||
|
# QSA indexer Q/K projection, replicated in a TP=4 deployment.
|
||||||
|
(640, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
},
|
||||||
|
# GDN and QSA output projections, TP=4.
|
||||||
|
(2560, 1536): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=1536),
|
||||||
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=1536),
|
||||||
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=1536),
|
||||||
|
},
|
||||||
|
# QSA fused QKV/gate projection, TP=4.
|
||||||
|
(3584, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||||
|
2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
||||||
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
},
|
||||||
|
# GDN fused QKVZ projection, TP=4.
|
||||||
|
(4096, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
||||||
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560),
|
||||||
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
||||||
|
},
|
||||||
|
# LM head, TP=4.
|
||||||
|
(62080, 2560): {
|
||||||
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_sm100() -> bool:
|
||||||
|
return current_platform.is_device_capability((10, 0))
|
||||||
|
|
||||||
|
|
||||||
def _is_sm103() -> bool:
|
def _is_sm103() -> bool:
|
||||||
return current_platform.is_device_capability((10, 3))
|
return current_platform.is_device_capability((10, 3))
|
||||||
|
|
||||||
@@ -155,6 +222,8 @@ def _is_sm90() -> bool:
|
|||||||
def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]:
|
def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]:
|
||||||
if _is_sm103():
|
if _is_sm103():
|
||||||
return QWEN4_EXP_GEMM_PLANS
|
return QWEN4_EXP_GEMM_PLANS
|
||||||
|
if _is_sm100():
|
||||||
|
return QWEN4_EXP_SM100_GEMM_PLANS
|
||||||
if _is_sm90():
|
if _is_sm90():
|
||||||
return QWEN4_EXP_SM90_GEMM_PLANS
|
return QWEN4_EXP_SM90_GEMM_PLANS
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
Reference in New Issue
Block a user