[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
6a9109d865
commit
c810937573
@@ -8,6 +8,8 @@ import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
_PREFETCH_TILES = 2
|
||||
|
||||
|
||||
class CuteSkinnyGemm:
|
||||
"""Shape-dynamic low-latency GEMM for small token counts.
|
||||
@@ -15,8 +17,9 @@ class CuteSkinnyGemm:
|
||||
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
|
||||
inputs, FP32 accumulators, and an output matching the input dtype. The
|
||||
residual term is optional and is added before the output conversion. N and
|
||||
K are runtime values. The tiny M dimension is fully unrolled alongside a
|
||||
small set of tuning parameters.
|
||||
K are runtime values unless the configuration specializes K at compile
|
||||
time. The tiny M dimension is fully unrolled alongside a small set of
|
||||
tuning parameters.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -30,9 +33,15 @@ class CuteSkinnyGemm:
|
||||
k_unroll: int = 1,
|
||||
has_residual: bool = False,
|
||||
use_pdl: bool = False,
|
||||
static_k: int | None = None,
|
||||
) -> None:
|
||||
if block_size % cute.arch.WARP_SIZE != 0:
|
||||
raise ValueError("block_size must be a multiple of the warp size")
|
||||
k_tile_size = block_size * vector_width
|
||||
if static_k is not None and (
|
||||
static_k % k_tile_size or static_k < _PREFETCH_TILES * k_tile_size
|
||||
):
|
||||
raise ValueError("static K must contain at least two complete tiles")
|
||||
self.element_type = element_type
|
||||
self.num_rows = num_rows
|
||||
self.block_size = block_size
|
||||
@@ -41,6 +50,7 @@ class CuteSkinnyGemm:
|
||||
self.k_unroll = k_unroll
|
||||
self.has_residual = has_residual
|
||||
self.use_pdl = use_pdl
|
||||
self.static_k = static_k
|
||||
self.num_warps = block_size // cute.arch.WARP_SIZE
|
||||
|
||||
@cute.jit
|
||||
@@ -66,7 +76,22 @@ class CuteSkinnyGemm:
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
|
||||
)
|
||||
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
|
||||
copy_b_prefetch = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
self.element_type,
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
|
||||
)
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gResidual,
|
||||
gC,
|
||||
k,
|
||||
copy_a,
|
||||
copy_b,
|
||||
copy_b_prefetch,
|
||||
).launch(
|
||||
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
|
||||
block=[self.block_size, 1, 1],
|
||||
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
|
||||
@@ -85,6 +110,7 @@ class CuteSkinnyGemm:
|
||||
k_extent: cutlass.Int32,
|
||||
copy_a: cute.CopyAtom,
|
||||
copy_b: cute.CopyAtom,
|
||||
copy_b_prefetch: cute.CopyAtom,
|
||||
) -> None:
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
block_idx, _, _ = cute.arch.block_idx()
|
||||
@@ -102,12 +128,8 @@ class CuteSkinnyGemm:
|
||||
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
n_base = block_idx * outputs_per_block
|
||||
k_tile_size: cutlass.Constexpr = block_size * vector_width
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
|
||||
gA_vec = cute.logical_divide(gA, (None, vector_width))
|
||||
gB_vec = cute.logical_divide(gB, (None, vector_width))
|
||||
@@ -127,21 +149,72 @@ class CuteSkinnyGemm:
|
||||
self.element_type,
|
||||
)
|
||||
|
||||
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
if const_expr(self.static_k is not None):
|
||||
num_k_tiles: cutlass.Constexpr = self.static_k // k_tile_size
|
||||
prefetched_b = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(_PREFETCH_TILES, outputs_per_block, vector_width),
|
||||
stride=(outputs_per_block * vector_width, vector_width, 1),
|
||||
),
|
||||
self.element_type,
|
||||
)
|
||||
for k_tile in cutlass.range_constexpr(_PREFETCH_TILES):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b_prefetch,
|
||||
tB_static[None, k_tile],
|
||||
prefetched_b[k_tile, ni, None],
|
||||
)
|
||||
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
tB = tB_all[n_idx, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for k_tile in cutlass.range_constexpr(num_k_tiles):
|
||||
if const_expr(k_tile < _PREFETCH_TILES):
|
||||
b_f32 = prefetched_b[k_tile, None, None].load().to(cutlass.Float32)
|
||||
else:
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
|
||||
cutlass.Float32
|
||||
) * b_regs[ni, vi].to(cutlass.Float32)
|
||||
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b,
|
||||
tB_static[None, k_tile],
|
||||
b_regs[ni, None],
|
||||
)
|
||||
b_f32 = b_regs.load().to(cutlass.Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
a_f32 = a_regs.load().to(cutlass.Float32)
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] += a_f32[mi, vi] * b_f32[ni, vi]
|
||||
else:
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
tB_dynamic = tB_all[n_idx, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b,
|
||||
tB_dynamic[None, k_tile],
|
||||
b_regs[ni, None],
|
||||
)
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
|
||||
cutlass.Float32
|
||||
) * b_regs[ni, vi].to(cutlass.Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
|
||||
@@ -23,6 +23,7 @@ class SkinnyGemmConfig:
|
||||
outputs_per_block: int
|
||||
k_unroll: int = 1
|
||||
vector_width: int = 8
|
||||
static_k: int | None = None
|
||||
|
||||
|
||||
class ShapeDynamicSkinnyGemm:
|
||||
@@ -112,7 +113,11 @@ class ShapeDynamicSkinnyGemm:
|
||||
|
||||
element_type = self._cutlass_dtype(dtype)
|
||||
n = cute.sym_int(divisibility=config.outputs_per_block)
|
||||
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
k = (
|
||||
config.static_k
|
||||
if config.static_k is not None
|
||||
else cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
)
|
||||
a = make_fake_tensor(
|
||||
element_type,
|
||||
(config.num_rows, k),
|
||||
@@ -134,6 +139,7 @@ class ShapeDynamicSkinnyGemm:
|
||||
k_unroll=config.k_unroll,
|
||||
has_residual=has_residual,
|
||||
use_pdl=self._use_pdl(),
|
||||
static_k=config.static_k,
|
||||
)
|
||||
self._compiled[(dtype, config, has_residual)] = cute.compile(
|
||||
kernel,
|
||||
@@ -239,6 +245,8 @@ class ShapeDynamicSkinnyGemm:
|
||||
raise ValueError(
|
||||
"K must be divisible by block_size * vector_width for this config"
|
||||
)
|
||||
if config.static_k is not None and a.shape[1] != config.static_k:
|
||||
raise ValueError("input K must match config static_k")
|
||||
has_residual = residual is not None
|
||||
cache_key = (a.dtype, config, has_residual)
|
||||
if cache_key not in self._compiled:
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
Backend = Literal["cute", "dsv3_fused_a"]
|
||||
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GLM52ProjectionSpec:
|
||||
n: int
|
||||
k: int
|
||||
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...]
|
||||
dsv3_tokens: frozenset[int] = frozenset()
|
||||
|
||||
def build_plan(self) -> dict[int, ResolvedCall]:
|
||||
plan: dict[int, ResolvedCall] = {
|
||||
num_tokens: ("cute", config) for num_tokens, config in self.cute_configs
|
||||
}
|
||||
plan.update(
|
||||
(num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens
|
||||
)
|
||||
return plan
|
||||
|
||||
|
||||
GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec(
|
||||
n=2624,
|
||||
k=6144,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 128, 4, static_k=6144)),
|
||||
(2, SkinnyGemmConfig(2, 128, 2)),
|
||||
),
|
||||
dsv3_tokens=frozenset(range(3, 17)),
|
||||
)
|
||||
|
||||
GLM52_Q_B_PROJECTION = GLM52ProjectionSpec(
|
||||
n=2048,
|
||||
k=2048,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 128, 4, static_k=2048)),
|
||||
(2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)),
|
||||
),
|
||||
dsv3_tokens=frozenset(range(3, 17)),
|
||||
)
|
||||
|
||||
# The MTP eh_proj is a plain nn.Linear, so it gets its plan through
|
||||
# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4.
|
||||
GLM52_EH_PROJECTION = GLM52ProjectionSpec(
|
||||
n=6144,
|
||||
k=12288,
|
||||
cute_configs=(
|
||||
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)),
|
||||
(2, SkinnyGemmConfig(2, 64, 2)),
|
||||
(3, SkinnyGemmConfig(3, 64, 2)),
|
||||
),
|
||||
)
|
||||
|
||||
GLM52_PROJECTIONS = {
|
||||
(spec.n, spec.k): spec
|
||||
for spec in (
|
||||
GLM52_QKV_A_PROJECTION,
|
||||
GLM52_Q_B_PROJECTION,
|
||||
GLM52_EH_PROJECTION,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _is_sm103() -> bool:
|
||||
return current_platform.is_device_capability((10, 3))
|
||||
|
||||
|
||||
def _is_supported_row_major(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
||||
|
||||
|
||||
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and _is_supported_row_major(x)
|
||||
and _is_supported_row_major(weight)
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and x.is_cuda
|
||||
and weight.is_cuda
|
||||
and x.device == weight.device
|
||||
and x.shape[1] == weight.shape[1]
|
||||
)
|
||||
|
||||
|
||||
def run_glm52_plan(
|
||||
plan: dict[int, ResolvedCall] | None,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
if plan is None or not _runtime_ok(x, weight):
|
||||
return None
|
||||
entry = plan.get(x.shape[0])
|
||||
if entry is None:
|
||||
return None
|
||||
|
||||
backend, config = entry
|
||||
if backend == "cute":
|
||||
if not shape_dynamic_skinny_gemm.is_available():
|
||||
return None
|
||||
return shape_dynamic_skinny_gemm(x, weight, config)
|
||||
|
||||
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
||||
return None
|
||||
output = torch.empty(
|
||||
(x.shape[0], weight.shape[0]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
|
||||
return output
|
||||
|
||||
|
||||
def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None:
|
||||
if configs and shape_dynamic_skinny_gemm.is_available():
|
||||
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs)
|
||||
|
||||
|
||||
def build_glm52_plan(
|
||||
weight: torch.Tensor | None, dtype: torch.dtype
|
||||
) -> dict[int, ResolvedCall] | None:
|
||||
"""Plan for a weight the walk below cannot reach (a plain ``nn.Linear``)."""
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return None
|
||||
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
|
||||
return None
|
||||
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
|
||||
if spec is None:
|
||||
return None
|
||||
_request_warmup(dtype, {config for _, config in spec.cute_configs})
|
||||
return spec.build_plan()
|
||||
|
||||
|
||||
class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod):
|
||||
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
|
||||
super().__init__()
|
||||
self._plan = plan
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if bias is None:
|
||||
output = run_glm52_plan(self._plan, x, layer.weight)
|
||||
if output is not None:
|
||||
return output
|
||||
return super().apply(layer, x, bias)
|
||||
|
||||
|
||||
def enable_glm52_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return
|
||||
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
for child in module.modules():
|
||||
if (
|
||||
not isinstance(child, LinearBase)
|
||||
or type(child.quant_method) is not UnquantizedLinearMethod
|
||||
):
|
||||
continue
|
||||
weight = getattr(child, "weight", None)
|
||||
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
|
||||
continue
|
||||
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
|
||||
if spec is None:
|
||||
continue
|
||||
child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan())
|
||||
warmup_configs.update(config for _, config in spec.cute_configs)
|
||||
|
||||
_request_warmup(dtype, warmup_configs)
|
||||
@@ -42,6 +42,8 @@ from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
|
||||
|
||||
|
||||
def _all_gather_sp_states(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -405,6 +407,11 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
|
||||
|
||||
model_cls = DeepseekV32Model
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
if self.config.model_type == "glm_moe_dsa":
|
||||
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def set_moe_parameters(self):
|
||||
# Same as the base, but keyed on the MoE block type rather than the
|
||||
# decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module).
|
||||
|
||||
@@ -42,6 +42,11 @@ from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .glm52_low_latency_gemm import (
|
||||
build_glm52_plan,
|
||||
enable_glm52_low_latency_gemm,
|
||||
run_glm52_plan,
|
||||
)
|
||||
from .model import DeepseekV32DecoderLayer
|
||||
|
||||
|
||||
@@ -56,6 +61,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
self._eh_plan = (
|
||||
build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype)
|
||||
if config.model_type == "glm_moe_dsa"
|
||||
else None
|
||||
)
|
||||
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
@@ -91,7 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
|
||||
if hidden_states is None:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
@@ -199,6 +211,8 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
self.model = DeepseekV32MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if self.config.model_type == "glm_moe_dsa":
|
||||
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self):
|
||||
|
||||
Reference in New Issue
Block a user