Signed-off-by: TimRen01 <HolaTim@hotmail.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
421 lines
18 KiB
Python
421 lines
18 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Qwen4Exp decode GEMM selection on Hopper and Blackwell.
|
|
|
|
Dispatch follows Kimi-K3 and uses the local ``(N, K)`` shape and token count.
|
|
Plans contain measured CUDA graph capture sizes; other token counts use the
|
|
standard linear implementation.
|
|
"""
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
import vllm.envs as envs
|
|
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
|
SkinnyGemmConfig,
|
|
row_stride_ok,
|
|
shape_dynamic_skinny_gemm,
|
|
)
|
|
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
|
ParallelLMHead,
|
|
UnquantizedEmbeddingMethod,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.torch_utils import direct_register_custom_op
|
|
|
|
QWEN4_EXP_SM103_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
|
# GDN fused QKVZ projection, TP=4.
|
|
(4096, 2560): {
|
|
1: SkinnyGemmConfig(1, 64, 4, k_unroll=4),
|
|
2: SkinnyGemmConfig(2, 64, 4, k_unroll=4),
|
|
},
|
|
# GDN and QSA output projections, TP=4.
|
|
(2560, 1536): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=2, vector_width=4),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2),
|
|
},
|
|
# GDN fused B/A projection, TP=4.
|
|
(24, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
|
|
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4),
|
|
16: SkinnyGemmConfig(16, 128, 1, k_unroll=4, vector_width=4),
|
|
},
|
|
# QSA fused QKV/gate + replicated indexer Q/K, TP=4 (GB300).
|
|
(4224, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 4, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 4, vector_width=4, static_k=2560),
|
|
},
|
|
# Shared-expert fused gate/up projection, TP=4.
|
|
(320, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
|
|
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
|
|
8: SkinnyGemmConfig(8, 64, 1, k_unroll=4),
|
|
16: SkinnyGemmConfig(16, 128, 2, k_unroll=4, vector_width=4),
|
|
},
|
|
# LM head, TP=4.
|
|
(62080, 2560): {
|
|
1: SkinnyGemmConfig(1, 64, 4, k_unroll=2),
|
|
2: SkinnyGemmConfig(2, 32, 4, k_unroll=2),
|
|
},
|
|
}
|
|
|
|
# H200 plans selected by exhaustive CUDA graph replay measurements over
|
|
# M={1, 2, 4, 8, 16}. Only points that beat the standard linear implementation
|
|
# in both hot-cache and L2-flush measurements are retained; other token counts
|
|
# keep the standard implementation and its GEMM heuristics.
|
|
QWEN4_EXP_SM90_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
|
# GDN fused QKVZ projection, TP=4.
|
|
(4096, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 4, vector_width=4, static_k=2560),
|
|
},
|
|
# QSA fused QKV/gate + replicated indexer Q/K, TP=4.
|
|
(4224, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=6, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=3, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 4, k_unroll=1, vector_width=8),
|
|
},
|
|
# GDN and QSA output projections, TP=4.
|
|
(2560, 1536): {
|
|
1: SkinnyGemmConfig(1, 128, 4, vector_width=2, static_k=1536),
|
|
2: SkinnyGemmConfig(2, 128, 4, vector_width=4, static_k=1536),
|
|
4: SkinnyGemmConfig(4, 64, 4, k_unroll=6, vector_width=4),
|
|
},
|
|
# GDN fused B/A projection, TP=4.
|
|
(24, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 3, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 1, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
|
},
|
|
# Shared-expert fused gate/up projection, TP=4.
|
|
(320, 2560): {
|
|
1: SkinnyGemmConfig(1, 64, 2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 4, k_unroll=5, vector_width=4),
|
|
4: SkinnyGemmConfig(4, 160, 1, k_unroll=2),
|
|
8: SkinnyGemmConfig(8, 128, 1, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
|
},
|
|
# LM head, TP=4.
|
|
(62080, 2560): {
|
|
1: SkinnyGemmConfig(1, 64, 2, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 2, vector_width=2, static_k=2560),
|
|
},
|
|
# Final HC down projection, replicated in a TP=4 deployment.
|
|
(320, 10240): {
|
|
1: SkinnyGemmConfig(1, 256, 1, static_k=10240),
|
|
2: SkinnyGemmConfig(2, 128, 1, k_unroll=10),
|
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=10),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=10),
|
|
},
|
|
}
|
|
|
|
|
|
# B200 plans; the winning configs differ from the B300 table.
|
|
QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
|
# GDN fused B/A projection, TP=4.
|
|
(24, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
},
|
|
# Shared-expert fused gate/up projection, TP=4.
|
|
(320, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 160, 1, k_unroll=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
|
},
|
|
# Final HC down projection, replicated in a TP=4 deployment.
|
|
(320, 10240): {
|
|
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240),
|
|
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240),
|
|
4: SkinnyGemmConfig(4, 256, 1, k_unroll=2),
|
|
8: SkinnyGemmConfig(8, 256, 1),
|
|
},
|
|
# GDN and QSA output projections, TP=4.
|
|
(2560, 1536): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=1536),
|
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=1536),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=1536),
|
|
},
|
|
# QSA fused QKV/gate + replicated indexer Q/K, TP=4.
|
|
(4224, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 4, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 4, vector_width=4, static_k=2560),
|
|
},
|
|
# GDN fused QKVZ projection, TP=4.
|
|
(4096, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
|
},
|
|
# LM head, TP=4.
|
|
(62080, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
|
},
|
|
}
|
|
|
|
|
|
# DGX Spark (GB10) plans for TP=1 and TP=2 local shapes.
|
|
QWEN4_EXP_SM121_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
|
# Shared-expert gate.
|
|
(1, 2560): {
|
|
1: SkinnyGemmConfig(1, 64, 1, k_unroll=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 1, vector_width=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
|
|
},
|
|
# GDN fused B/A projection, TP=2.
|
|
(48, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
},
|
|
# GDN fused B/A projection.
|
|
(96, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 4, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 4, vector_width=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 64, 1),
|
|
},
|
|
# Router.
|
|
(512, 2560): {
|
|
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 1, k_unroll=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 64, 2, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 64, 2, k_unroll=2, static_k=2560),
|
|
},
|
|
# Shared-expert fused gate/up, TP=2; retain after indexer fusion.
|
|
(640, 2560): {
|
|
1: SkinnyGemmConfig(1, 256, 1, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 256, 1, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 64, 1, static_k=2560),
|
|
},
|
|
# Shared-expert fused gate/up projection.
|
|
(1280, 2560): {
|
|
1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 1, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 128, 1, k_unroll=2, vector_width=2, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, static_k=2560),
|
|
},
|
|
# Shared-expert down projection.
|
|
(2560, 640): {
|
|
1: SkinnyGemmConfig(1, 64, 1, k_unroll=4, vector_width=2, static_k=640),
|
|
2: SkinnyGemmConfig(2, 64, 1, vector_width=2, static_k=640),
|
|
4: SkinnyGemmConfig(4, 64, 1, vector_width=2, static_k=640),
|
|
8: SkinnyGemmConfig(8, 32, 1, k_unroll=4, vector_width=4, static_k=640),
|
|
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=4, static_k=640),
|
|
},
|
|
# GDN and QSA output projections, TP=2.
|
|
(2560, 3072): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=3072),
|
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=3072),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=3072),
|
|
},
|
|
# GDN and QSA output projections.
|
|
(2560, 6144): {
|
|
1: SkinnyGemmConfig(1, 64, 2, vector_width=4, static_k=6144),
|
|
2: SkinnyGemmConfig(2, 64, 1, k_unroll=4, vector_width=4, static_k=6144),
|
|
4: SkinnyGemmConfig(4, 64, 1, k_unroll=4, vector_width=4, static_k=6144),
|
|
8: SkinnyGemmConfig(8, 128, 1, vector_width=2, static_k=6144),
|
|
16: SkinnyGemmConfig(16, 32, 1, static_k=6144),
|
|
},
|
|
# QSA fused QKV/gate projection, TP=2 (LoRA keeps the indexer projection separate).
|
|
(6656, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
|
},
|
|
# QSA fused QKV/gate + replicated indexer Q/K projection, TP=2.
|
|
(7296, 2560): {
|
|
1: SkinnyGemmConfig(1, 32, 1, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 32, 1, vector_width=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 32, 4, vector_width=2, static_k=2560),
|
|
},
|
|
# GDN fused QKVZ projection, TP=2.
|
|
(8192, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560),
|
|
},
|
|
# HC up projection.
|
|
(10240, 320): {
|
|
1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, vector_width=2, static_k=320),
|
|
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=320),
|
|
4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=320),
|
|
8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=320),
|
|
},
|
|
# QSA fused QKV/gate + replicated indexer Q/K projection.
|
|
(13952, 2560): {
|
|
1: SkinnyGemmConfig(1, 32, 1, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 32, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
},
|
|
# GDN fused QKVZ projection.
|
|
(16384, 2560): {
|
|
1: SkinnyGemmConfig(1, 32, 1, k_unroll=4, vector_width=2, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=2, static_k=2560),
|
|
},
|
|
# LM head, TP=2.
|
|
(124160, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
|
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2),
|
|
},
|
|
# LM head.
|
|
(248320, 2560): {
|
|
1: SkinnyGemmConfig(1, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
2: SkinnyGemmConfig(2, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560),
|
|
8: SkinnyGemmConfig(8, 64, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
|
|
},
|
|
}
|
|
|
|
|
|
QWEN4_EXP_GEMM_PLANS_BY_CAPABILITY: dict[
|
|
tuple[int, int] | None, dict[tuple[int, int], dict[int, SkinnyGemmConfig]]
|
|
] = {
|
|
(10, 3): QWEN4_EXP_SM103_GEMM_PLANS,
|
|
(10, 0): QWEN4_EXP_SM100_GEMM_PLANS,
|
|
(9, 0): QWEN4_EXP_SM90_GEMM_PLANS,
|
|
(12, 1): QWEN4_EXP_SM121_GEMM_PLANS,
|
|
}
|
|
|
|
|
|
def _is_packed_row_major(tensor: torch.Tensor) -> bool:
|
|
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
|
|
|
|
|
def _runtime_ok(
|
|
x: torch.Tensor, weight: torch.Tensor, config: SkinnyGemmConfig
|
|
) -> bool:
|
|
return (
|
|
not envs.VLLM_BATCH_INVARIANT
|
|
and x.dim() == 2
|
|
and row_stride_ok(x, config)
|
|
and _is_packed_row_major(weight)
|
|
and x.dtype == torch.bfloat16
|
|
and weight.dtype == torch.bfloat16
|
|
and x.is_cuda
|
|
and weight.is_cuda
|
|
and x.device == weight.device
|
|
and x.shape[1] == weight.shape[1]
|
|
)
|
|
|
|
|
|
class _Qwen4ExpLowLatencyApply:
|
|
def apply(
|
|
self,
|
|
layer: nn.Module,
|
|
x: torch.Tensor,
|
|
bias: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
if bias is None and not envs.VLLM_BATCH_INVARIANT:
|
|
return torch.ops.vllm.qwen4_exp_low_latency_gemm(x, layer.weight)
|
|
return super().apply(layer, x, bias) # type: ignore[misc]
|
|
|
|
|
|
class Qwen4ExpLowLatencyLinearMethod(_Qwen4ExpLowLatencyApply, UnquantizedLinearMethod):
|
|
pass
|
|
|
|
|
|
class Qwen4ExpLowLatencyEmbeddingMethod(
|
|
_Qwen4ExpLowLatencyApply, UnquantizedEmbeddingMethod
|
|
):
|
|
pass
|
|
|
|
|
|
def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
|
plans = QWEN4_EXP_GEMM_PLANS_BY_CAPABILITY.get(
|
|
current_platform.get_device_capability(), {}
|
|
)
|
|
plan = plans.get((weight.shape[0], weight.shape[1]))
|
|
config = None if plan is None else plan.get(x.shape[0])
|
|
if (
|
|
config is not None
|
|
and _runtime_ok(x, weight, config)
|
|
and shape_dynamic_skinny_gemm.is_available()
|
|
):
|
|
return shape_dynamic_skinny_gemm(x, weight, config)
|
|
return torch.nn.functional.linear(x, weight)
|
|
|
|
|
|
def _qwen4_exp_low_latency_gemm_fake(
|
|
x: torch.Tensor, weight: torch.Tensor
|
|
) -> torch.Tensor:
|
|
return x.new_empty((*x.shape[:-1], weight.shape[0]))
|
|
|
|
|
|
direct_register_custom_op(
|
|
op_name="qwen4_exp_low_latency_gemm",
|
|
op_func=_qwen4_exp_low_latency_gemm,
|
|
fake_impl=_qwen4_exp_low_latency_gemm_fake,
|
|
)
|
|
|
|
|
|
def enable_qwen4_exp_low_latency_gemm(
|
|
module: nn.Module,
|
|
dtype: torch.dtype,
|
|
) -> None:
|
|
plans = QWEN4_EXP_GEMM_PLANS_BY_CAPABILITY.get(
|
|
current_platform.get_device_capability(), {}
|
|
)
|
|
if dtype != torch.bfloat16 or not plans:
|
|
return
|
|
if not shape_dynamic_skinny_gemm.is_available():
|
|
return
|
|
|
|
warmup_configs: set[SkinnyGemmConfig] = set()
|
|
for child in module.modules():
|
|
is_linear = (
|
|
isinstance(child, LinearBase)
|
|
and type(child.quant_method) is UnquantizedLinearMethod
|
|
)
|
|
is_head = (
|
|
isinstance(child, ParallelLMHead)
|
|
and type(child.quant_method) is UnquantizedEmbeddingMethod
|
|
)
|
|
if not (is_linear or is_head):
|
|
continue
|
|
weight = getattr(child, "weight", None)
|
|
if weight is None or weight.dim() != 2:
|
|
continue
|
|
plan = plans.get((weight.shape[0], weight.shape[1]))
|
|
if plan is None:
|
|
continue
|
|
if is_linear:
|
|
child.quant_method = Qwen4ExpLowLatencyLinearMethod()
|
|
else:
|
|
child.quant_method = Qwen4ExpLowLatencyEmbeddingMethod()
|
|
warmup_configs.update(plan.values())
|
|
|
|
if warmup_configs:
|
|
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs)
|