[Perf][Qwen4Exp] Fuse HC down projection and SiLU on NVIDIA (#58957)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-09-28 18:51:38 +08:00
committed by GitHub
parent 252279314b
commit d6cce94fd4
7 changed files with 1175 additions and 23 deletions
+30
View File
@@ -4,11 +4,13 @@
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -22,6 +24,13 @@ HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
LORA_RANK = 320
DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded
requires_sm90 = pytest.mark.skipif(
not current_platform.has_device_capability(90),
reason="fused HC down+SiLU requires SM90+",
)
def test_grouped_gemma_rmsnorm() -> None:
@@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
assert torch.equal(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm)
@requires_sm90
@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48])
def test_hc_down_silu_fused(num_tokens: int) -> None:
# Compare computed columns with the unfused ll_bf16 + hc_silu reference.
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(0)
x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
lora, injection = hc_down_silu(x, weight, LORA_RANK, HC)
down = ll_bf16_gemm(x, weight).to(torch.bfloat16)
torch.testing.assert_close(
lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01
)
torch.testing.assert_close(
injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01
)
+38 -23
View File
@@ -25,16 +25,19 @@ Typical usage inside a transformer decoder layer::
import torch
from torch import nn
import vllm.envs as envs
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.platforms import current_platform
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.cute_dsl.hc_down_silu import MAX_FUSED_M, hc_down_silu
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
@@ -58,9 +61,9 @@ class GatedResidual(nn.Module):
injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject), so
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
standard quant_method mechanism.
are vLLM Linear modules (merged replicated linear for down+inject).
Eligible down+inject projections use the fused SiLU GEMM; other projections
use their Linear module's GEMM dispatch.
"""
def __init__(
@@ -105,6 +108,12 @@ class GatedResidual(nn.Module):
return_bias=False,
disable_tp=True,
)
weight = self.input_mix_weight_down_block_inject.weight
self._use_hc_down_silu = (
weight.shape[1] % 8 == 0
and weight.dtype == torch.bfloat16
and current_platform.has_device_capability(90)
)
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
@@ -125,6 +134,30 @@ class GatedResidual(nn.Module):
return_bias=False,
)
def _down_and_inject(
self, xn: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Down projection + SiLU; also returns injection logits if combined."""
if not self.use_combine:
return hc_silu(self.input_mix_weight_down(xn), self.hc_count), None
use_fused = (
self._use_hc_down_silu
and not envs.VLLM_BATCH_INVARIANT
and 1 <= xn.shape[0] <= MAX_FUSED_M
)
if use_fused:
return hc_down_silu(
xn,
self.input_mix_weight_down_block_inject.weight,
self.lora_rank,
self.hc_count,
)
down_and_injection = self.input_mix_weight_down_block_inject(xn)
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
return hc_silu(lora, self.hc_count), injection
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
@@ -135,16 +168,7 @@ class GatedResidual(nn.Module):
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
@@ -172,16 +196,7 @@ class GatedResidual(nn.Module):
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
lora, injection = self._down_and_inject(xn)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
+11
View File
@@ -78,6 +78,7 @@ from vllm.v1.kv_cache_interface import MambaSpec
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
from .ple_layer import Qwen4ExpPLELayer
from .qsa import Qwen4ExpQSAAttention
@@ -686,6 +687,16 @@ class Qwen4ExpForCausalLM(
)
self.set_moe_parameters(self.model.layers)
enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype)
if self.model_config.dtype == torch.bfloat16:
# Precompile the fused HC down+SiLU kernels for every CUDA-graph
# capture size in the fused dispatch range, so no CuTe-DSL JIT
# happens during graph capture.
request_hc_down_silu_warmup(
vllm_config.compilation_config.cudagraph_capture_sizes or (),
self.config.hc_lowrank,
self.config.hc_count,
self.config.hidden_size * self.config.hc_count,
)
@staticmethod
def get_model_state_cls():
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,334 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py.
Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough
elsewhere, with the production bf16 rounding boundary) into the FMA kernel.
Output is bf16 instead of fp32.
"""
import cutlass
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
def _sigmoid_f32(x):
return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True)
class HcDownSiluFma:
"""BF16 GEMM kernel based on CTA-local FMA dot products, with the mHC SiLU
epilogue fused into the final store.
Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the fp32
accumulator to bf16 (production rounding boundary), then applies
SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through.
Output is bf16 [M, N].
See the base class docstring in
vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py for the
GEMM structure; only the epilogue differs. The compile key is
``(M, K, threadblock_size, rank, hc)``.
"""
def __init__(
self,
k: int,
threadblock_size: int = 128,
main_vec_width: int = 8,
tail_vec_width: int = 4,
prefetch_pdl_weights: bool = False,
rank: int = 320,
hc: int = 4,
):
self.threadblock_size = threadblock_size
self.main_vec_width = main_vec_width
self.tail_vec_width = tail_vec_width
self.prefetch_pdl_weights = prefetch_pdl_weights
self.rank = rank
self.hc = hc
self.num_warps = threadblock_size // cute.arch.WARP_SIZE
self._init_k_tiles(k)
self.main_prefetch_tiles = min(self.main_tiles, 8)
def _vectorized_elems(self, k_extent: int, vec_width: int) -> int:
vector_tile = vec_width * self.threadblock_size
return (k_extent // vector_tile) * vector_tile
def _init_k_tiles(self, k: int) -> None:
"""Split K into vector loops, scalar rounds, and ragged tail."""
self.k_main_elems = self._vectorized_elems(k, self.main_vec_width)
self.k_after_main = k - self.k_main_elems
self.k_tail_elems = self._vectorized_elems(
self.k_after_main, self.tail_vec_width
)
self.k_done_all = self.k_main_elems + self.k_tail_elems
self.scalar_rem = k - self.k_done_all
self.ks_full = self.scalar_rem // self.threadblock_size
self.ks_part = self.scalar_rem % self.threadblock_size
self.k_scalar_full = self.ks_full * self.threadblock_size
self.k_part_offset = self.k_done_all + self.k_scalar_full
self.main_tiles = self.k_main_elems // (
self.main_vec_width * self.threadblock_size
)
self.tail_tiles = self.k_tail_elems // (
self.tail_vec_width * self.threadblock_size
)
@cute.jit
def _vector_fma(
self,
acc: cute.Tensor,
tA: cute.Tensor,
tB: cute.Tensor,
M: cutlass.Constexpr,
num_tiles: cutlass.Constexpr,
align_bytes: cutlass.Constexpr,
):
for tile in cutlass.range_constexpr(num_tiles):
bt = tB[None, tile]
br = cute.make_rmem_tensor_like(bt)
cute.autovec_copy(bt, br)
br_f32 = br.load().to(cutlass.Float32)
for m in cutlass.range_constexpr(M):
at = tA[m, None, tile]
ar = cute.make_rmem_tensor_like(at)
cute.autovec_copy(at, ar)
vec_width: cutlass.Constexpr = cute.size(ar)
for v in cutlass.range_constexpr(vec_width):
acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v]
@cute.jit
def _vector_fma_prefetched(
self,
acc: cute.Tensor,
tA: cute.Tensor,
tB: cute.Tensor,
rB: cute.Tensor,
M: cutlass.Constexpr,
num_tiles: cutlass.Constexpr,
prefetch_tiles: cutlass.Constexpr,
):
for tile in cutlass.range_constexpr(num_tiles):
if const_expr(tile < prefetch_tiles):
br_f32 = rB[tile, None].load().to(cutlass.Float32)
else:
bt = tB[None, tile]
br = cute.make_rmem_tensor_like(bt)
cute.autovec_copy(bt, br)
br_f32 = br.load().to(cutlass.Float32)
for m in cutlass.range_constexpr(M):
at = tA[m, None, tile]
ar = cute.make_rmem_tensor_like(at)
cute.autovec_copy(at, ar)
vec_width: cutlass.Constexpr = cute.size(ar)
for v in cutlass.range_constexpr(vec_width):
acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v]
def _make_thread_vector_slice(
self,
gA_vec: cute.Tensor,
gB_vec: cute.Tensor,
tidx: cutlass.Int32,
n_idx: cutlass.Int32,
threadblock_size: cutlass.Constexpr,
):
# (M/N, K_TILE, K_LANE, K_VEC); tidx selects K_LANE.
tA = cute.logical_divide(gA_vec, (None, (None, threadblock_size)))
tB = cute.logical_divide(gB_vec, (None, (None, threadblock_size)))
return tA[None, (None, (tidx, None))], tB[n_idx, (None, (tidx, None))]
def _make_k_slice(
self,
gX: cute.Tensor,
k_offset: cutlass.Constexpr,
k_extent: cutlass.Constexpr,
):
k_layout_extent: cutlass.Constexpr = (
1 if const_expr(k_extent == 0) else k_extent
)
if const_expr(k_offset == 0):
return cute.local_tile(
gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0)
)
return cute.local_tile(
cute.domain_offset((0, k_offset), gX),
(cute.size(gX, mode=[0]), k_layout_extent),
(0, 0),
)
@cute.jit
def __call__(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gC: cute.Tensor,
M: cutlass.Constexpr,
K_dim: cutlass.Constexpr,
N_dim: cutlass.Int32,
stream: CUstream,
):
self.kernel(
gA,
gB,
gC,
M,
self.main_vec_width,
self.tail_vec_width,
self.threadblock_size,
self.num_warps,
self.k_main_elems,
self.k_tail_elems,
self.k_done_all,
self.ks_full,
self.ks_part,
self.k_scalar_full,
self.k_part_offset,
self.main_tiles,
self.tail_tiles,
self.rank,
self.hc,
).launch(
grid=[N_dim, 1, 1],
block=[self.threadblock_size, 1, 1],
smem=M * 4 * self.num_warps,
stream=stream,
use_pdl=True,
min_blocks_per_mp=1,
)
@cute.kernel
def kernel(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gC: cute.Tensor,
M: cutlass.Constexpr,
main_vec_width: cutlass.Constexpr,
tail_vec_width: cutlass.Constexpr,
threadblock_size: cutlass.Constexpr,
num_warps: cutlass.Constexpr,
k_main_elems: cutlass.Constexpr,
k_tail_elems: cutlass.Constexpr,
k_done_all: cutlass.Constexpr,
ks_full: cutlass.Constexpr,
ks_part: cutlass.Constexpr,
k_scalar_full: cutlass.Constexpr,
k_part_offset: cutlass.Constexpr,
main_tiles: cutlass.Constexpr,
tail_tiles: cutlass.Constexpr,
rank: cutlass.Constexpr,
hc: cutlass.Constexpr,
):
tidx, _, _ = cute.arch.thread_idx()
n_idx, _, _ = cute.arch.block_idx()
wid = cute.arch.warp_idx()
# One FP32 accumulator per token.
acc = cute.make_rmem_tensor((M,), cutlass.Float32)
acc.fill(0.0)
if const_expr(k_main_elems > 0):
gA_main = self._make_k_slice(gA, 0, k_main_elems)
gB_main = self._make_k_slice(gB, 0, k_main_elems)
gA_vec = cute.logical_divide(gA_main, (None, main_vec_width))
gB_vec = cute.logical_divide(gB_main, (None, main_vec_width))
tA, tB = self._make_thread_vector_slice(
gA_vec, gB_vec, tidx, n_idx, threadblock_size
)
if const_expr(self.prefetch_pdl_weights):
prefetch_tiles: cutlass.Constexpr = self.main_prefetch_tiles
prefetched_b = cute.make_rmem_tensor(
cute.make_layout(
(prefetch_tiles, main_vec_width),
stride=(main_vec_width, 1),
),
cutlass.BFloat16,
)
for tile in cutlass.range_constexpr(prefetch_tiles):
cute.autovec_copy(tB[None, tile], prefetched_b[tile, None])
cute.arch.griddepcontrol_wait()
# 128-bit vectorized main loop
if const_expr(k_main_elems > 0):
if const_expr(self.prefetch_pdl_weights):
self._vector_fma_prefetched(
acc,
tA,
tB,
prefetched_b,
M,
main_tiles,
prefetch_tiles,
)
else:
self._vector_fma(acc, tA, tB, M, main_tiles, 16)
# 64-bit vectorized tail (K remainder after main loop)
if const_expr(k_tail_elems > 0):
gA_tail = self._make_k_slice(gA, k_main_elems, k_tail_elems)
gB_tail = self._make_k_slice(gB, k_main_elems, k_tail_elems)
gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width))
gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width))
tA_t, tB_t = self._make_thread_vector_slice(
gA_tail_vec, gB_tail_vec, tidx, n_idx, threadblock_size
)
self._vector_fma(acc, tA_t, tB_t, M, tail_tiles, 8)
# Full scalar rounds use CuTe width-1 tiles; KS_PART is the ragged tail.
if const_expr(ks_full > 0):
gA_scalar = self._make_k_slice(gA, k_done_all, k_scalar_full)
gB_scalar = self._make_k_slice(gB, k_done_all, k_scalar_full)
gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1))
gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1))
tA_s, tB_s = self._make_thread_vector_slice(
gA_scalar_vec, gB_scalar_vec, tidx, n_idx, threadblock_size
)
self._vector_fma(acc, tA_s, tB_s, M, ks_full, 2)
# Only threads below KS_PART load the ragged tail.
if const_expr(ks_part > 0):
gA_part = self._make_k_slice(gA, k_part_offset, ks_part)
gB_part = self._make_k_slice(gB, k_part_offset, ks_part)
if tidx < ks_part:
bv2 = gB_part[n_idx, tidx].to(cutlass.Float32)
for m in cutlass.range_constexpr(M):
acc[m] = acc[m] + gA_part[m, tidx].to(cutlass.Float32) * bv2
# Intra-warp shuffle reduction
for m in cutlass.range_constexpr(M):
acc[m] = cute.arch.warp_reduction_sum(acc[m])
# Cross-warp reduction via shared memory
smem_red_layout = cute.make_layout((M, num_warps), stride=(num_warps, 1))
smem = cutlass.utils.SmemAllocator()
sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16)
with cute.arch.elect_one():
for m in cutlass.range_constexpr(M):
sm[m, wid] = acc[m]
# Final reduction, fused mHC SiLU epilogue, and output.
cute.arch.sync_threads()
if tidx == 0:
for m in cutlass.range_constexpr(M):
partials = sm[m, None].load()
total = cutlass.Float32(
partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
)
# Production rounding boundary: GEMM output is materialized as
# bf16 before _hc_silu_kernel reads it.
x_bf16 = total.to(cutlass.BFloat16)
if n_idx < rank:
z = x_bf16.to(cutlass.Float32) * (1.0 / hc)
gC[m, n_idx] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16)
else:
gC[m, n_idx] = x_bf16
cute.arch.griddepcontrol_launch_dependents()
@@ -0,0 +1,536 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py.
Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough
elsewhere, with the production bf16 rounding boundary) into the split-K kernel,
applied after the cluster reduction where the final fp32 result materializes.
Output is bf16 instead of fp32.
"""
import math
import cutlass
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass._mlir import ir as _ir
from cutlass._mlir.dialects import llvm as _llvm
from cutlass.cutlass_dsl import dsl_user_op
from cutlass.pipeline import sm90 as pipeline
# Map a local smem address to the same peer CTA DSMEM offset.
@dsl_user_op
def set_block_rank(smem_ptr, peer_rank, *, loc=None, ip=None):
dsmem_ptr = cute.arch.map_dsmem_ptr(smem_ptr, peer_rank, loc=loc, ip=ip)
return cutlass.Int32(dsmem_ptr.toint(loc=loc, ip=ip))
# Plain DSMEM store; mbarrier helpers do not model this reduction.
@dsl_user_op
def st_shared_remote_f32(remote_addr, val, *, loc=None, ip=None):
i32 = _ir.IntegerType.get_signless(32)
addr_ir = remote_addr.ir_value(loc=loc, ip=ip)
val_ir = val.ir_value(loc=loc, ip=ip)
_llvm.inline_asm(
i32,
[addr_ir, val_ir],
"st.shared::cluster.f32 [$0], $1; mov.u32 $2, 0;",
"r,f,=r",
has_side_effects=True, # keep the inline store ordered
loc=loc,
ip=ip,
)
def _sigmoid_f32(x):
return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True)
class HcDownSiluMma:
"""BF16 GEMM kernel based on clustered split-K MMA, with the mHC SiLU
epilogue fused into the final store.
Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the reduced
fp32 accumulator to bf16 (production rounding boundary), then applies
SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through.
Output is bf16 [M, N].
See the base class docstring in
vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py for the
GEMM structure; only the epilogue differs. The compile key is
``(split_k, num_stages, tile_n, rank, hc)``.
"""
def __init__(
self,
ab_dtype=cutlass.BFloat16,
acc_dtype=cutlass.Float32,
tile_n: int = 16,
tile_k: int = 256,
num_stages: int = 2,
num_dma_warps: int = 4,
split_k: int = 8,
rank: int = 320,
hc: int = 4,
):
self.ab_dtype = ab_dtype
self.acc_dtype = acc_dtype
self.rank = rank
self.hc = hc
self.tile_m = 16
self.tile_n = tile_n
self.tile_k = tile_k
self.copy_bits = 128
self.num_stages = num_stages
self.split_k = split_k
self.mma_shape = (16, 8, 16) # mma.sync.aligned.m16n8k16
self.atom_layout = (1, 1, 1) # one MMA atom per warp
self.num_dma_warps = num_dma_warps
self.num_mma_warps = 4
self.num_dma_threads = self.num_dma_warps * cute.arch.WARP_SIZE
self.num_mma_threads = self.num_mma_warps * cute.arch.WARP_SIZE
self.num_threads = self.num_dma_threads + self.num_mma_threads
self.num_epilogue_elems = self.tile_m * self.tile_n
self.epilogue_elems_per_thread = self.num_epilogue_elems // self.num_mma_threads
def _make_smem_layout_AB(self, dtype, copy_bits, smem_tiler):
"""Build the staged swizzled SMEM layout for A or B tiles."""
major_size = min(smem_tiler[1], 64)
# Match swizzle span to contiguous K bytes, capped by CuTe 3-bit swizzle.
swizzle_bits = int(math.log2(major_size * dtype.width // copy_bits))
swizzle_bits = min(swizzle_bits, 3)
# Tile the swizzled atom across (M_or_N, K, stages).
layout_atom_outer = cute.make_layout((8, major_size), stride=(major_size, 1))
layout_atom = cute.make_composed_layout(
cute.make_swizzle(swizzle_bits, 3, 3), 0, layout_atom_outer
)
return cute.tile_to_shape(layout_atom, smem_tiler, (0, 1, 2))
def _make_gmem_tiled_copy(self, atom_copy, dtype, copy_bits, num_threads):
"""Build the per-thread cp.async vector-copy layout."""
# Lay threads across K so each lane issues one vector copy.
copy_elems = copy_bits // dtype.width
k_threads = cute.size(self.tile_k) // copy_elems # threads along K
thread_layout = cute.make_layout(
(num_threads // k_threads, k_threads), stride=(k_threads, 1)
)
value_layout = cute.make_layout((1, copy_elems))
return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout)
@cute.jit
def _fill_pred(self, pred_flat, coord_tensor, k_tile, dim_limit, K_total):
# Predicate one K tile and keep the stage-broadcast view in sync.
coord_ktile = coord_tensor[None, None, 0, k_tile]
num_vec = pred_flat.shape[0]
num_mn = pred_flat.shape[1]
for v in cutlass.range_constexpr(num_vec):
# pred_flat is (K_VEC, M/N) for one K_TILE.
for j in cutlass.range_constexpr(num_mn):
pred_flat[v, j] = cute.elem_less(
coord_ktile[(0, v), j], (dim_limit, K_total)
)
def _make_pred(self, tXcX, k_tile, dim_limit, K_total):
# pred_flat is (K_VEC, M/N); pred is (K_VEC, M/N, STAGE).
num_vec = tXcX.shape[0][1]
num_mn = cute.size(tXcX, mode=[1])
pred_flat = cute.make_rmem_tensor(
cute.make_layout((num_vec, num_mn), stride=(num_mn, 1)),
cutlass.Boolean,
)
pred = cute.make_tensor(
pred_flat.iterator,
cute.make_layout(
(num_vec, num_mn, cute.size(tXcX, mode=[2])),
stride=(num_mn, 1, 0),
),
)
return pred_flat, pred
@cute.jit
def __call__(
self,
mA: cute.Tensor,
mB: cute.Tensor,
mC: cute.Tensor,
stream: CUstream,
):
bM, bN, bK = self.tile_m, self.tile_n, self.tile_k
copy_bits: cutlass.Constexpr = self.copy_bits
sA_layout = self._make_smem_layout_AB(
mA.element_type, copy_bits, (bM, bK, self.num_stages)
)
sB_layout = self._make_smem_layout_AB(
mB.element_type, copy_bits, (bN, bK, self.num_stages)
)
@cute.struct
class SharedStorage:
a: cute.struct.Align[
cute.struct.MemRange[mA.element_type, cute.cosize(sA_layout)], 16
]
b: cute.struct.Align[
cute.struct.MemRange[mB.element_type, cute.cosize(sB_layout)], 16
]
mbar: cute.struct.Align[
cute.struct.MemRange[cutlass.Int64, self.num_stages * 2], 8
]
atom_g2s = cute.make_copy_atom(
cute.nvgpu.cpasync.CopyG2SOp(
cache_mode=cute.nvgpu.cpasync.LoadCacheMode.GLOBAL
),
mA.element_type,
num_bits_per_copy=copy_bits,
) # cp.async GMEM -> SMEM, bypassing L1
tiled_copy_A = self._make_gmem_tiled_copy(
atom_g2s, mA.element_type, copy_bits, self.num_dma_threads
)
tiled_copy_B = self._make_gmem_tiled_copy(
atom_g2s, mB.element_type, copy_bits, self.num_dma_threads
)
op = cute.nvgpu.warp.MmaF16BF16Op(self.ab_dtype, self.acc_dtype, self.mma_shape)
# Repeat the m16n8k16 atom along N to cover the CTA output tile.
perm_mnk = (
self.atom_layout[0] * self.mma_shape[0],
self.atom_layout[1] * self.mma_shape[1] * (self.tile_n // 8),
self.atom_layout[2] * self.mma_shape[2],
)
tiled_mma = cute.make_tiled_mma(
op, cute.make_layout(self.atom_layout), permutation_mnk=perm_mnk
)
tiler_mn = (bM, bN)
grid_m, grid_n = cute.ceil_div(mC.shape, tiler_mn)
self.kernel(
mA,
mB,
mC,
sA_layout,
sB_layout,
tiled_copy_A,
tiled_copy_B,
tiled_mma,
SharedStorage,
self.rank,
self.hc,
).launch(
grid=[
cute.size(grid_m),
cute.size(grid_n),
self.split_k,
],
block=[self.num_threads, 1, 1],
cluster=[
1,
1,
self.split_k,
], # split-K CTAs form one cluster
stream=stream,
use_pdl=True,
)
@cute.kernel
def kernel(
self,
mA,
mB,
mC,
sA_layout: cute.ComposedLayout,
sB_layout: cute.ComposedLayout,
tiled_copy_A: cute.TiledCopy,
tiled_copy_B: cute.TiledCopy,
tiled_mma: cute.TiledMma,
shared_storage: cutlass.Constexpr,
rank: cutlass.Constexpr,
hc: cutlass.Constexpr,
):
bM, bN, bK = self.tile_m, self.tile_n, self.tile_k
num_stages = self.num_stages
tidx, _, _ = cute.arch.thread_idx()
bid_m, bid_n, bid_z = cute.arch.block_idx()
warp_idx = cute.arch.warp_idx()
lane_id = cute.arch.lane_idx()
num_dma_warps: cutlass.Constexpr = self.num_dma_warps
is_dma_warp = warp_idx < num_dma_warps
dma_tidx = tidx
mma_tidx = tidx - self.num_dma_threads
mma_warp_idx = warp_idx - num_dma_warps
cta_tiler = (bM, bN, bK)
coord = (bid_m, bid_n, None) # all K tiles
# CTA-local tiles.
gA = cute.local_tile(
mA, tiler=cta_tiler, coord=coord, proj=(1, None, 1)
) # skip N
gB = cute.local_tile(
mB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1)
) # skip M
gC = cute.local_tile(
mC, tiler=cta_tiler, coord=coord, proj=(1, 1, None)
) # skip K
mcA = cute.make_identity_tensor(mA.layout.shape)
mcB = cute.make_identity_tensor(mB.layout.shape)
mcC = cute.make_identity_tensor(mC.layout.shape)
# Coordinate modes: cA=(M,K,k_tile), cB=(N,K,k_tile), cC=(M,N).
cA = cute.local_tile(mcA, tiler=cta_tiler, coord=coord, proj=(1, None, 1))
cB = cute.local_tile(mcB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1))
cC = cute.local_tile(mcC, tiler=cta_tiler, coord=coord, proj=(1, 1, None))
# 128-bit cp.async copies require 16-byte aligned GMEM views.
gA = cute.make_tensor(gA.iterator.align(16), gA.layout)
gB = cute.make_tensor(gB.iterator.align(16), gB.layout)
smem = cutlass.utils.SmemAllocator()
storage_ptr = smem.allocate(shared_storage.size_in_bytes(), byte_alignment=16) # type: ignore[attr-defined]
storage = shared_storage(storage_ptr) # type: ignore[call-arg]
sA = storage.a.get_tensor(sA_layout)
sB = storage.b.get_tensor(sB_layout)
# Pipeline cp.async producers into MMA consumers.
producer_group = pipeline.CooperativeGroup(
pipeline.Agent.Thread, self.num_dma_threads
)
consumer_group = pipeline.CooperativeGroup(
pipeline.Agent.Thread, self.num_mma_threads
)
mainloop_pipeline = pipeline.PipelineCpAsync.create(
barrier_storage=storage.mbar.data_ptr(),
num_stages=num_stages,
producer_group=producer_group,
consumer_group=consumer_group,
)
# Round-robin split-K: split z handles tiles z, z+split_k, ...
K_total = cute.size(mA, mode=[1])
k_tile_count = cute.size(gA, mode=[2])
k_start = bid_z
num_k_tiles = cute.ceil_div(k_tile_count - k_start, self.split_k)
if is_dma_warp:
# DMA warps trade registers for copy throughput.
cute.arch.setmaxregister_decrease(40)
thr_A = tiled_copy_A.get_slice(dma_tidx)
thr_B = tiled_copy_B.get_slice(dma_tidx)
tAgA = thr_A.partition_S(gA)
tAsA = thr_A.partition_D(sA)
tBgB = thr_B.partition_S(gB)
tBsB = thr_B.partition_D(sB)
tAcA = thr_A.partition_S(cA)
tBcB = thr_B.partition_S(cB)
# Build M/K and N/K predicates, broadcast across K-tile copies.
tApA_flat, tApA = self._make_pred(tAcA, k_start, mA.shape[0], K_total)
tBpB_flat, tBpB = self._make_pred(tBcB, k_start, mB.shape[0], K_total)
self._fill_pred(tApA_flat, tAcA, k_start, mA.shape[0], K_total)
self._fill_pred(tBpB_flat, tBcB, k_start, mB.shape[0], K_total)
producer_state = pipeline.make_pipeline_state(
pipeline.PipelineUserType.Producer, num_stages
)
# Prime the first pipeline stage.
mainloop_pipeline.producer_acquire(producer_state)
cute.copy(
tiled_copy_B,
tBgB[None, None, None, k_start],
tBsB[None, None, None, producer_state.index],
pred=tBpB,
)
cute.arch.griddepcontrol_wait()
cute.copy(
tiled_copy_A,
tAgA[None, None, None, k_start],
tAsA[None, None, None, producer_state.index],
pred=tApA,
)
mainloop_pipeline.producer_commit(producer_state)
producer_state.advance()
for k_tile in cutlass.range(
k_start + self.split_k, k_tile_count, self.split_k, unroll=1
):
self._fill_pred(tApA_flat, tAcA, k_tile, mA.shape[0], K_total)
self._fill_pred(tBpB_flat, tBcB, k_tile, mB.shape[0], K_total)
mainloop_pipeline.producer_acquire(producer_state)
cute.copy(
tiled_copy_A,
tAgA[None, None, None, k_tile],
tAsA[None, None, None, producer_state.index],
pred=tApA,
)
cute.copy(
tiled_copy_B,
tBgB[None, None, None, k_tile],
tBsB[None, None, None, producer_state.index],
pred=tBpB,
)
mainloop_pipeline.producer_commit(producer_state)
producer_state.advance()
mainloop_pipeline.producer_tail(producer_state)
else:
# MMA warps with k-phase interleaving
cute.arch.setmaxregister_increase(232) # large MMA fragments
num_mma_warps: cutlass.Constexpr = self.num_mma_warps
thr_mma = tiled_mma.get_slice(lane_id)
tCsA = thr_mma.partition_A(sA)
tCsB = thr_mma.partition_B(sB)
tCgC = thr_mma.partition_C(gC)
tCrA = tiled_mma.make_fragment_A(tCsA[None, None, None, 0])
tCrB = tiled_mma.make_fragment_B(tCsB[None, None, None, 0])
tCrC = tiled_mma.make_fragment_C(tCgC)
tCrC.fill(0.0)
# ldmatrix moves SMEM fragments into MMA registers.
atom_s2r_A = cute.make_copy_atom(
cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mA.element_type
) # non-transposed, x4
atom_s2r_B = cute.make_copy_atom(
cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mB.element_type
)
# SMEM -> register copy path.
tiled_s2r_A = cute.make_tiled_copy_A(atom_s2r_A, tiled_mma)
tiled_s2r_B = cute.make_tiled_copy_B(atom_s2r_B, tiled_mma)
thr_s2r_A = tiled_s2r_A.get_slice(lane_id)
thr_s2r_B = tiled_s2r_B.get_slice(lane_id)
tCsA_v = thr_s2r_A.partition_S(sA) # views, not copies
tCrA_v = thr_s2r_A.retile(tCrA)
tCsB_v = thr_s2r_B.partition_S(sB)
tCrB_v = thr_s2r_B.retile(tCrB)
# Split the MMA K-fragments across the MMA warps.
tCsA_warp_v = cute.logical_divide(tCsA_v, (None, None, num_mma_warps, None))
tCsB_warp_v = cute.logical_divide(tCsB_v, (None, None, num_mma_warps, None))
num_k_blocks = cute.size(tCrA, mode=[2])
k_blocks_per_warp: cutlass.Constexpr = num_k_blocks // num_mma_warps
consumer_state = pipeline.make_pipeline_state(
pipeline.PipelineUserType.Consumer, num_stages
)
# Shape-dynamic split-K count, so this stays a runtime range.
for _ in cutlass.range(num_k_tiles, unroll_full=True):
mainloop_pipeline.consumer_wait(consumer_state)
for ki in cutlass.range_constexpr(k_blocks_per_warp):
cute.copy(
tiled_s2r_A,
tCsA_warp_v[
None, None, (mma_warp_idx, ki), consumer_state.index
],
tCrA_v[None, None, 0],
) # ldmatrix
cute.copy(
tiled_s2r_B,
tCsB_warp_v[
None, None, (mma_warp_idx, ki), consumer_state.index
],
tCrB_v[None, None, 0],
)
cute.gemm(
tiled_mma, tCrC, tCrA[None, None, 0], tCrB[None, None, 0], tCrC
) # mma.sync
mainloop_pipeline.consumer_release(consumer_state)
consumer_state.advance()
# Cluster reduction epilogue.
# Reduce per-warp accumulators within this CTA.
num_elems: cutlass.Constexpr = self.num_epilogue_elems
elems_per_thread: cutlass.Constexpr = self.epilogue_elems_per_thread
# Map MMA threads to linear CTA output elements.
epilogue_thread_layout = cute.make_layout(
(elems_per_thread, self.num_mma_threads),
stride=(self.num_mma_threads, 1),
)
epilogue_slots = cute.make_tensor(0, epilogue_thread_layout)
epilogue_slot_coords = cute.make_identity_tensor((bN, bM))
# Layout: (mma_warp, linear MN element).
smem_red = cute.make_tensor(
cute.arch.alloc_smem(
cutlass.Float32, num_elems * num_mma_warps, alignment=16
),
cute.make_layout((num_mma_warps, num_elems), stride=(num_elems, 1)),
)
smem_warp = cute.make_tensor(
cute.domain_offset((mma_warp_idx, 0), smem_red).iterator,
cute.make_layout((bM, bN), stride=(bN, 1)),
)
tCsC_partial = thr_mma.partition_C(smem_warp)
cute.autovec_copy(tCrC, tCsC_partial)
cute.arch.sync_threads()
# Layout: (split-K rank, linear MN element).
partials = cute.make_tensor(
cute.arch.alloc_smem(
cutlass.Float32, num_elems * self.split_k, alignment=16
),
cute.make_layout((self.split_k, num_elems), stride=(num_elems, 1)),
)
cta_rank = cute.arch.block_idx_in_cluster()
for ei in cutlass.range_constexpr(elems_per_thread):
elem_idx = epilogue_slots[ei, mma_tidx]
local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0])
total = cutlass.Float32(0.0)
if cute.elem_less(local_coord, gC.shape):
total = (
smem_red[None, elem_idx]
.load()
.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
)
partials[cta_rank, elem_idx] = total
cute.arch.sync_threads()
# Broadcast this CTA's partials to peer DSMEM.
for ei in cutlass.range_constexpr(elems_per_thread):
elem_idx = epilogue_slots[ei, mma_tidx]
my_slot = cute.domain_offset((cta_rank, elem_idx), partials).iterator
my_val = partials[cta_rank, elem_idx]
for peer in cutlass.range_constexpr(self.split_k):
remote = set_block_rank(my_slot, cutlass.Int32(peer))
st_shared_remote_f32(remote, my_val)
cute.arch.cluster_arrive()
cute.arch.cluster_wait() # peer DSMEM stores are now visible
if mma_tidx == 0:
cute.arch.griddepcontrol_launch_dependents()
cute.arch.sync_threads()
# Reduce split-K partials, apply the fused mHC SiLU epilogue, and
# write the bf16 global output.
for ei in cutlass.range_constexpr(elems_per_thread):
elem_idx = epilogue_slots[ei, mma_tidx]
local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0])
global_coord = cC[local_coord]
if cute.elem_less(global_coord, mC.shape):
acc = cutlass.Float32(
partials[None, elem_idx]
.load()
.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
)
# Production rounding boundary: the GEMM result is
# materialized as bf16 before _hc_silu_kernel reads it.
x_bf16 = acc.to(cutlass.BFloat16)
if global_coord[1] < rank:
z = x_bf16.to(cutlass.Float32) * (1.0 / hc)
gC[local_coord] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16)
else:
gC[local_coord] = x_bf16
cute.arch.sync_threads()
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py.
Fused Qwen4Exp mHC down projection + SiLU epilogue, where
columns < lora_rank get ``silu(bf16(acc) / hc_count)``, the hc_count
injection-logit columns pass through as ``bf16(acc)``, and the pad columns
are never computed. Output is bf16 (ll_bf16's fp32-output bonus is given up
to preserve the production rounding boundary).
The GEMM uses an FMA backend for M <= 4 and a split-K MMA backend beyond,
both with PDL. The model uses its Linear module above M=48.
"""
import logging
from collections.abc import Iterable
from functools import lru_cache, partial
from typing import Any, Literal
import torch
from vllm.platforms import current_platform
logger = logging.getLogger(__name__)
# The fused kernel stops winning past M ~ 64 on Qwen3.8-Next-Flash.
MAX_FUSED_M = 48
# Split-K configs tuned on SM100f for Qwen3.8-Flash-Next (K=10240, N=324).
_SM100F_TUNED_SPLITK: dict[int, tuple[int, int, int]] = {
**{m: (6, 5, 8) for m in range(5, 17)},
**{m: (6, 5, 16) for m in range(17, 33)},
**{m: (6, 4, 16) for m in range(33, MAX_FUSED_M + 1)},
}
# FMA: (backend, M, K, threadblock_size); MMA: (backend, split_k, stages, tile_n).
CompileKey = tuple[Literal["fma", "mma"], int, int, int]
class HcDownSiluGemm:
"""Dispatch/compile cache for the fused mHC down+SiLU GEMM."""
def __init__(
self, rank: int, hc: int, k: int, *, prefetch_pdl_weights: bool = False
) -> None:
self.rank = rank
self.hc = hc
self.k = k
self._prefetch_pdl_weights = prefetch_pdl_weights
self._compiled: dict[CompileKey, Any] = {}
self._warmup_m: set[int] = set()
self._warmup_registered = False
def dispatch(self, m: int) -> CompileKey:
if m <= 4 or self.k < 2048:
return ("fma", m, self.k, 128)
return ("mma", *_SM100F_TUNED_SPLITK[m])
def compile(self, compile_key: CompileKey) -> None:
if compile_key in self._compiled:
return
import cutlass.cute as cute
from cutlass import BFloat16
from quack.compile_utils import make_fake_tensor
N = cute.sym_int()
gemm: Any
extra_args: tuple[int, ...]
if compile_key[0] == "mma":
from ._hc_down_silu_mma import HcDownSiluMma
_, split_k, num_stages, tile_n = compile_key
M, K = cute.sym_int(), cute.sym_int()
gemm = HcDownSiluMma(
tile_n=tile_n,
num_stages=num_stages,
split_k=split_k,
rank=self.rank,
hc=self.hc,
)
extra_args = ()
options = "--enable-tvm-ffi"
else:
from ._hc_down_silu_fma import HcDownSiluFma
_, M, K, threadblock_size = compile_key
gemm = HcDownSiluFma(
k=K,
threadblock_size=threadblock_size,
prefetch_pdl_weights=self._prefetch_pdl_weights,
rank=self.rank,
hc=self.hc,
)
extra_args = (M, K, 1) # runtime N placeholder
options = "--enable-tvm-ffi --ptxas-options -maxrregcount=64"
hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=8)
router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=8)
output = make_fake_tensor(BFloat16, (M, N), divisibility=1)
self._compiled[compile_key] = cute.compile(
gemm,
hidden_states,
router_weight,
output,
*extra_args,
cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
options=options,
)
logger.debug("Compiled hc_down_silu: %s", compile_key)
def request_warmup(self, m_values: Iterable[int]) -> None:
m_values = set(m_values)
if not m_values:
return
self._warmup_m.update(m_values)
if self._warmup_registered:
return
from vllm.model_executor.warmup.cutedsl_warmup import (
register_cutedsl_warmup_provider,
)
register_cutedsl_warmup_provider(self)
self._warmup_registered = True
def get_cutedsl_warmup_compile_units(self):
from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit
return tuple(
CuTeDSLCompileUnit(
name="Qwen4Exp HC down+SiLU GEMM",
key=(
"qwen4-exp-hc-down-silu",
self.rank,
self.hc,
self.k,
self._prefetch_pdl_weights,
compile_key,
),
compile=partial(self.compile, compile_key),
)
for compile_key in dict.fromkeys(
self.dispatch(m) for m in sorted(self._warmup_m)
)
)
def __call__(
self,
hidden_states: torch.Tensor, # [M, K] bf16
router_weight: torch.Tensor, # [N, K] bf16
) -> torch.Tensor: # [M, N] bf16
M = hidden_states.shape[0]
N = router_weight.shape[0]
n_compute = self.rank + self.hc
w_gemm = router_weight[:n_compute]
compile_key = self.dispatch(M)
self.compile(compile_key)
kernel = self._compiled[compile_key]
output = torch.empty(M, N, dtype=torch.bfloat16, device=hidden_states.device)
out_gemm = output[:, :n_compute]
if compile_key[0] == "mma":
kernel(hidden_states, w_gemm, out_gemm)
else:
kernel(hidden_states, w_gemm, out_gemm, n_compute)
return output
@lru_cache
def _get_kernel(
rank: int, hc: int, k: int, prefetch_pdl_weights: bool = False
) -> HcDownSiluGemm:
return HcDownSiluGemm(rank, hc, k, prefetch_pdl_weights=prefetch_pdl_weights)
def hc_down_silu(
x: torch.Tensor,
weight: torch.Tensor,
rank: int,
hc: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused Qwen4Exp HC down projection + SiLU.
Args:
x: Normalized hyper-hidden input, [M, K].
weight: Merged down+inject weight, [N, K].
rank: Number of low-rank output columns.
hc: Number of injection-logit output columns.
Returns:
LoRA activations [M, rank] and injection logits [M, hc].
"""
kernel = _get_kernel(rank, hc, weight.shape[1], x.shape[0] == 1)
output = kernel(x, weight)
return output[:, :rank], output[:, rank : rank + hc]
def request_hc_down_silu_warmup(
m_values: Iterable[int], rank: int, hc: int, k: int
) -> None:
"""Precompile the fused kernels for CUDA-graph capture token counts.
Args:
m_values: Token counts the model may capture CUDA graphs for; values
outside the fused dispatch range are ignored.
rank: Number of low-rank output columns.
hc: Number of injection-logit output columns.
k: Input feature size.
"""
if not current_platform.has_device_capability(90) or k % 8 != 0:
return
m_set = {int(m) for m in m_values if 1 <= m <= MAX_FUSED_M}
# M=1 dispatches to the weight-prefetching PDL variant.
_get_kernel(rank, hc, k, True).request_warmup(m_set & {1})
_get_kernel(rank, hc, k).request_warmup(m_set - {1})
__all__ = [
"MAX_FUSED_M",
"hc_down_silu",
"request_hc_down_silu_warmup",
]