[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-08-03 16:49:41 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 6a9109d865
commit c810937573
7 changed files with 619 additions and 51 deletions
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
import torch
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.platforms import current_platform
Backend = Literal["cute", "dsv3_fused_a"]
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
@dataclass(frozen=True, slots=True)
class GLM52ProjectionSpec:
n: int
k: int
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...]
dsv3_tokens: frozenset[int] = frozenset()
def build_plan(self) -> dict[int, ResolvedCall]:
plan: dict[int, ResolvedCall] = {
num_tokens: ("cute", config) for num_tokens, config in self.cute_configs
}
plan.update(
(num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens
)
return plan
GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec(
n=2624,
k=6144,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=6144)),
(2, SkinnyGemmConfig(2, 128, 2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
GLM52_Q_B_PROJECTION = GLM52ProjectionSpec(
n=2048,
k=2048,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=2048)),
(2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
# The MTP eh_proj is a plain nn.Linear, so it gets its plan through
# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4.
GLM52_EH_PROJECTION = GLM52ProjectionSpec(
n=6144,
k=12288,
cute_configs=(
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)),
(2, SkinnyGemmConfig(2, 64, 2)),
(3, SkinnyGemmConfig(3, 64, 2)),
),
)
GLM52_PROJECTIONS = {
(spec.n, spec.k): spec
for spec in (
GLM52_QKV_A_PROJECTION,
GLM52_Q_B_PROJECTION,
GLM52_EH_PROJECTION,
)
}
def _is_sm103() -> bool:
return current_platform.is_device_capability((10, 3))
def _is_supported_row_major(tensor: torch.Tensor) -> bool:
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
not envs.VLLM_BATCH_INVARIANT
and _is_supported_row_major(x)
and _is_supported_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and x.is_cuda
and weight.is_cuda
and x.device == weight.device
and x.shape[1] == weight.shape[1]
)
def run_glm52_plan(
plan: dict[int, ResolvedCall] | None,
x: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor | None:
if plan is None or not _runtime_ok(x, weight):
return None
entry = plan.get(x.shape[0])
if entry is None:
return None
backend, config = entry
if backend == "cute":
if not shape_dynamic_skinny_gemm.is_available():
return None
return shape_dynamic_skinny_gemm(x, weight, config)
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
return None
output = torch.empty(
(x.shape[0], weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
return output
def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None:
if configs and shape_dynamic_skinny_gemm.is_available():
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs)
def build_glm52_plan(
weight: torch.Tensor | None, dtype: torch.dtype
) -> dict[int, ResolvedCall] | None:
"""Plan for a weight the walk below cannot reach (a plain ``nn.Linear``)."""
if dtype != torch.bfloat16 or not _is_sm103():
return None
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
return None
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
return None
_request_warmup(dtype, {config for _, config in spec.cute_configs})
return spec.build_plan()
class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod):
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
super().__init__()
self._plan = plan
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if bias is None:
output = run_glm52_plan(self._plan, x, layer.weight)
if output is not None:
return output
return super().apply(layer, x, bias)
def enable_glm52_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
if dtype != torch.bfloat16 or not _is_sm103():
return
warmup_configs: set[SkinnyGemmConfig] = set()
for child in module.modules():
if (
not isinstance(child, LinearBase)
or type(child.quant_method) is not UnquantizedLinearMethod
):
continue
weight = getattr(child, "weight", None)
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
continue
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
continue
child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan())
warmup_configs.update(config for _, config in spec.cute_configs)
_request_warmup(dtype, warmup_configs)
+7
View File
@@ -42,6 +42,8 @@ from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.sequence import IntermediateTensors
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
def _all_gather_sp_states(
hidden_states: torch.Tensor,
@@ -405,6 +407,11 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
model_cls = DeepseekV32Model
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
def set_moe_parameters(self):
# Same as the base, but keyed on the MoE block type rather than the
# decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module).
+15 -1
View File
@@ -42,6 +42,11 @@ from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .glm52_low_latency_gemm import (
build_glm52_plan,
enable_glm52_low_latency_gemm,
run_glm52_plan,
)
from .model import DeepseekV32DecoderLayer
@@ -56,6 +61,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self._eh_plan = (
build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype)
if config.model_type == "glm_moe_dsa"
else None
)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
@@ -91,7 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.hnorm.weight,
self.enorm.variance_epsilon,
)
hidden_states = self.eh_proj(eh_input)
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
if hidden_states is None:
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
@@ -199,6 +211,8 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
self.model = DeepseekV32MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
self.set_moe_parameters()
def set_moe_parameters(self):