[Perf][Qwen4Exp] Fuse HC down projection and SiLU on NVIDIA (#58957)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -4,11 +4,13 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu
|
||||
from vllm.models.qwen4_exp.nvidia.ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
@@ -22,6 +24,13 @@ HC = 4
|
||||
HIDDEN_SIZE = 2560
|
||||
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
|
||||
EPS = 1e-6
|
||||
LORA_RANK = 320
|
||||
DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90),
|
||||
reason="fused HC down+SiLU requires SM90+",
|
||||
)
|
||||
|
||||
|
||||
def test_grouped_gemma_rmsnorm() -> None:
|
||||
@@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
|
||||
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
|
||||
assert torch.equal(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48])
|
||||
def test_hc_down_silu_fused(num_tokens: int) -> None:
|
||||
# Compare computed columns with the unfused ll_bf16 + hc_silu reference.
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
lora, injection = hc_down_silu(x, weight, LORA_RANK, HC)
|
||||
|
||||
down = ll_bf16_gemm(x, weight).to(torch.bfloat16)
|
||||
torch.testing.assert_close(
|
||||
lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01
|
||||
)
|
||||
|
||||
@@ -25,16 +25,19 @@ Typical usage inside a transformer decoder layer::
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..common.hyperconnection import (
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from .ops.cute_dsl.hc_down_silu import MAX_FUSED_M, hc_down_silu
|
||||
from .ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
@@ -58,9 +61,9 @@ class GatedResidual(nn.Module):
|
||||
injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
|
||||
standard quant_method mechanism.
|
||||
are vLLM Linear modules (merged replicated linear for down+inject).
|
||||
Eligible down+inject projections use the fused SiLU GEMM; other projections
|
||||
use their Linear module's GEMM dispatch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -105,6 +108,12 @@ class GatedResidual(nn.Module):
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
weight = self.input_mix_weight_down_block_inject.weight
|
||||
self._use_hc_down_silu = (
|
||||
weight.shape[1] % 8 == 0
|
||||
and weight.dtype == torch.bfloat16
|
||||
and current_platform.has_device_capability(90)
|
||||
)
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
@@ -125,6 +134,30 @@ class GatedResidual(nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def _down_and_inject(
|
||||
self, xn: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Down projection + SiLU; also returns injection logits if combined."""
|
||||
if not self.use_combine:
|
||||
return hc_silu(self.input_mix_weight_down(xn), self.hc_count), None
|
||||
|
||||
use_fused = (
|
||||
self._use_hc_down_silu
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and 1 <= xn.shape[0] <= MAX_FUSED_M
|
||||
)
|
||||
if use_fused:
|
||||
return hc_down_silu(
|
||||
xn,
|
||||
self.input_mix_weight_down_block_inject.weight,
|
||||
self.lora_rank,
|
||||
self.hc_count,
|
||||
)
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
return hc_silu(lora, self.hc_count), injection
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
@@ -135,16 +168,7 @@ class GatedResidual(nn.Module):
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
lora, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
@@ -172,16 +196,7 @@ class GatedResidual(nn.Module):
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
lora, injection = self._down_and_inject(xn)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
@@ -686,6 +687,16 @@ class Qwen4ExpForCausalLM(
|
||||
)
|
||||
self.set_moe_parameters(self.model.layers)
|
||||
enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype)
|
||||
if self.model_config.dtype == torch.bfloat16:
|
||||
# Precompile the fused HC down+SiLU kernels for every CUDA-graph
|
||||
# capture size in the fused dispatch range, so no CuTe-DSL JIT
|
||||
# happens during graph capture.
|
||||
request_hc_down_silu_warmup(
|
||||
vllm_config.compilation_config.cudagraph_capture_sizes or (),
|
||||
self.config.hc_lowrank,
|
||||
self.config.hc_count,
|
||||
self.config.hidden_size * self.config.hc_count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_model_state_cls():
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,334 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py.
|
||||
|
||||
Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough
|
||||
elsewhere, with the production bf16 rounding boundary) into the FMA kernel.
|
||||
Output is bf16 instead of fp32.
|
||||
"""
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
|
||||
def _sigmoid_f32(x):
|
||||
return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True)
|
||||
|
||||
|
||||
class HcDownSiluFma:
|
||||
"""BF16 GEMM kernel based on CTA-local FMA dot products, with the mHC SiLU
|
||||
epilogue fused into the final store.
|
||||
|
||||
Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the fp32
|
||||
accumulator to bf16 (production rounding boundary), then applies
|
||||
SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through.
|
||||
Output is bf16 [M, N].
|
||||
|
||||
See the base class docstring in
|
||||
vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py for the
|
||||
GEMM structure; only the epilogue differs. The compile key is
|
||||
``(M, K, threadblock_size, rank, hc)``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
k: int,
|
||||
threadblock_size: int = 128,
|
||||
main_vec_width: int = 8,
|
||||
tail_vec_width: int = 4,
|
||||
prefetch_pdl_weights: bool = False,
|
||||
rank: int = 320,
|
||||
hc: int = 4,
|
||||
):
|
||||
self.threadblock_size = threadblock_size
|
||||
self.main_vec_width = main_vec_width
|
||||
self.tail_vec_width = tail_vec_width
|
||||
self.prefetch_pdl_weights = prefetch_pdl_weights
|
||||
self.rank = rank
|
||||
self.hc = hc
|
||||
self.num_warps = threadblock_size // cute.arch.WARP_SIZE
|
||||
self._init_k_tiles(k)
|
||||
self.main_prefetch_tiles = min(self.main_tiles, 8)
|
||||
|
||||
def _vectorized_elems(self, k_extent: int, vec_width: int) -> int:
|
||||
vector_tile = vec_width * self.threadblock_size
|
||||
return (k_extent // vector_tile) * vector_tile
|
||||
|
||||
def _init_k_tiles(self, k: int) -> None:
|
||||
"""Split K into vector loops, scalar rounds, and ragged tail."""
|
||||
self.k_main_elems = self._vectorized_elems(k, self.main_vec_width)
|
||||
self.k_after_main = k - self.k_main_elems
|
||||
self.k_tail_elems = self._vectorized_elems(
|
||||
self.k_after_main, self.tail_vec_width
|
||||
)
|
||||
self.k_done_all = self.k_main_elems + self.k_tail_elems
|
||||
self.scalar_rem = k - self.k_done_all
|
||||
self.ks_full = self.scalar_rem // self.threadblock_size
|
||||
self.ks_part = self.scalar_rem % self.threadblock_size
|
||||
self.k_scalar_full = self.ks_full * self.threadblock_size
|
||||
self.k_part_offset = self.k_done_all + self.k_scalar_full
|
||||
self.main_tiles = self.k_main_elems // (
|
||||
self.main_vec_width * self.threadblock_size
|
||||
)
|
||||
self.tail_tiles = self.k_tail_elems // (
|
||||
self.tail_vec_width * self.threadblock_size
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _vector_fma(
|
||||
self,
|
||||
acc: cute.Tensor,
|
||||
tA: cute.Tensor,
|
||||
tB: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
num_tiles: cutlass.Constexpr,
|
||||
align_bytes: cutlass.Constexpr,
|
||||
):
|
||||
for tile in cutlass.range_constexpr(num_tiles):
|
||||
bt = tB[None, tile]
|
||||
br = cute.make_rmem_tensor_like(bt)
|
||||
cute.autovec_copy(bt, br)
|
||||
br_f32 = br.load().to(cutlass.Float32)
|
||||
|
||||
for m in cutlass.range_constexpr(M):
|
||||
at = tA[m, None, tile]
|
||||
ar = cute.make_rmem_tensor_like(at)
|
||||
cute.autovec_copy(at, ar)
|
||||
vec_width: cutlass.Constexpr = cute.size(ar)
|
||||
for v in cutlass.range_constexpr(vec_width):
|
||||
acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v]
|
||||
|
||||
@cute.jit
|
||||
def _vector_fma_prefetched(
|
||||
self,
|
||||
acc: cute.Tensor,
|
||||
tA: cute.Tensor,
|
||||
tB: cute.Tensor,
|
||||
rB: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
num_tiles: cutlass.Constexpr,
|
||||
prefetch_tiles: cutlass.Constexpr,
|
||||
):
|
||||
for tile in cutlass.range_constexpr(num_tiles):
|
||||
if const_expr(tile < prefetch_tiles):
|
||||
br_f32 = rB[tile, None].load().to(cutlass.Float32)
|
||||
else:
|
||||
bt = tB[None, tile]
|
||||
br = cute.make_rmem_tensor_like(bt)
|
||||
cute.autovec_copy(bt, br)
|
||||
br_f32 = br.load().to(cutlass.Float32)
|
||||
for m in cutlass.range_constexpr(M):
|
||||
at = tA[m, None, tile]
|
||||
ar = cute.make_rmem_tensor_like(at)
|
||||
cute.autovec_copy(at, ar)
|
||||
vec_width: cutlass.Constexpr = cute.size(ar)
|
||||
for v in cutlass.range_constexpr(vec_width):
|
||||
acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v]
|
||||
|
||||
def _make_thread_vector_slice(
|
||||
self,
|
||||
gA_vec: cute.Tensor,
|
||||
gB_vec: cute.Tensor,
|
||||
tidx: cutlass.Int32,
|
||||
n_idx: cutlass.Int32,
|
||||
threadblock_size: cutlass.Constexpr,
|
||||
):
|
||||
# (M/N, K_TILE, K_LANE, K_VEC); tidx selects K_LANE.
|
||||
tA = cute.logical_divide(gA_vec, (None, (None, threadblock_size)))
|
||||
tB = cute.logical_divide(gB_vec, (None, (None, threadblock_size)))
|
||||
return tA[None, (None, (tidx, None))], tB[n_idx, (None, (tidx, None))]
|
||||
|
||||
def _make_k_slice(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
k_offset: cutlass.Constexpr,
|
||||
k_extent: cutlass.Constexpr,
|
||||
):
|
||||
k_layout_extent: cutlass.Constexpr = (
|
||||
1 if const_expr(k_extent == 0) else k_extent
|
||||
)
|
||||
|
||||
if const_expr(k_offset == 0):
|
||||
return cute.local_tile(
|
||||
gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0)
|
||||
)
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((0, k_offset), gX),
|
||||
(cute.size(gX, mode=[0]), k_layout_extent),
|
||||
(0, 0),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
K_dim: cutlass.Constexpr,
|
||||
N_dim: cutlass.Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gC,
|
||||
M,
|
||||
self.main_vec_width,
|
||||
self.tail_vec_width,
|
||||
self.threadblock_size,
|
||||
self.num_warps,
|
||||
self.k_main_elems,
|
||||
self.k_tail_elems,
|
||||
self.k_done_all,
|
||||
self.ks_full,
|
||||
self.ks_part,
|
||||
self.k_scalar_full,
|
||||
self.k_part_offset,
|
||||
self.main_tiles,
|
||||
self.tail_tiles,
|
||||
self.rank,
|
||||
self.hc,
|
||||
).launch(
|
||||
grid=[N_dim, 1, 1],
|
||||
block=[self.threadblock_size, 1, 1],
|
||||
smem=M * 4 * self.num_warps,
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
min_blocks_per_mp=1,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
main_vec_width: cutlass.Constexpr,
|
||||
tail_vec_width: cutlass.Constexpr,
|
||||
threadblock_size: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
k_main_elems: cutlass.Constexpr,
|
||||
k_tail_elems: cutlass.Constexpr,
|
||||
k_done_all: cutlass.Constexpr,
|
||||
ks_full: cutlass.Constexpr,
|
||||
ks_part: cutlass.Constexpr,
|
||||
k_scalar_full: cutlass.Constexpr,
|
||||
k_part_offset: cutlass.Constexpr,
|
||||
main_tiles: cutlass.Constexpr,
|
||||
tail_tiles: cutlass.Constexpr,
|
||||
rank: cutlass.Constexpr,
|
||||
hc: cutlass.Constexpr,
|
||||
):
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
n_idx, _, _ = cute.arch.block_idx()
|
||||
wid = cute.arch.warp_idx()
|
||||
|
||||
# One FP32 accumulator per token.
|
||||
acc = cute.make_rmem_tensor((M,), cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
if const_expr(k_main_elems > 0):
|
||||
gA_main = self._make_k_slice(gA, 0, k_main_elems)
|
||||
gB_main = self._make_k_slice(gB, 0, k_main_elems)
|
||||
gA_vec = cute.logical_divide(gA_main, (None, main_vec_width))
|
||||
gB_vec = cute.logical_divide(gB_main, (None, main_vec_width))
|
||||
tA, tB = self._make_thread_vector_slice(
|
||||
gA_vec, gB_vec, tidx, n_idx, threadblock_size
|
||||
)
|
||||
if const_expr(self.prefetch_pdl_weights):
|
||||
prefetch_tiles: cutlass.Constexpr = self.main_prefetch_tiles
|
||||
prefetched_b = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(prefetch_tiles, main_vec_width),
|
||||
stride=(main_vec_width, 1),
|
||||
),
|
||||
cutlass.BFloat16,
|
||||
)
|
||||
for tile in cutlass.range_constexpr(prefetch_tiles):
|
||||
cute.autovec_copy(tB[None, tile], prefetched_b[tile, None])
|
||||
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
# 128-bit vectorized main loop
|
||||
if const_expr(k_main_elems > 0):
|
||||
if const_expr(self.prefetch_pdl_weights):
|
||||
self._vector_fma_prefetched(
|
||||
acc,
|
||||
tA,
|
||||
tB,
|
||||
prefetched_b,
|
||||
M,
|
||||
main_tiles,
|
||||
prefetch_tiles,
|
||||
)
|
||||
else:
|
||||
self._vector_fma(acc, tA, tB, M, main_tiles, 16)
|
||||
|
||||
# 64-bit vectorized tail (K remainder after main loop)
|
||||
if const_expr(k_tail_elems > 0):
|
||||
gA_tail = self._make_k_slice(gA, k_main_elems, k_tail_elems)
|
||||
gB_tail = self._make_k_slice(gB, k_main_elems, k_tail_elems)
|
||||
gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width))
|
||||
gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width))
|
||||
tA_t, tB_t = self._make_thread_vector_slice(
|
||||
gA_tail_vec, gB_tail_vec, tidx, n_idx, threadblock_size
|
||||
)
|
||||
self._vector_fma(acc, tA_t, tB_t, M, tail_tiles, 8)
|
||||
|
||||
# Full scalar rounds use CuTe width-1 tiles; KS_PART is the ragged tail.
|
||||
if const_expr(ks_full > 0):
|
||||
gA_scalar = self._make_k_slice(gA, k_done_all, k_scalar_full)
|
||||
gB_scalar = self._make_k_slice(gB, k_done_all, k_scalar_full)
|
||||
gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1))
|
||||
gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1))
|
||||
tA_s, tB_s = self._make_thread_vector_slice(
|
||||
gA_scalar_vec, gB_scalar_vec, tidx, n_idx, threadblock_size
|
||||
)
|
||||
self._vector_fma(acc, tA_s, tB_s, M, ks_full, 2)
|
||||
|
||||
# Only threads below KS_PART load the ragged tail.
|
||||
if const_expr(ks_part > 0):
|
||||
gA_part = self._make_k_slice(gA, k_part_offset, ks_part)
|
||||
gB_part = self._make_k_slice(gB, k_part_offset, ks_part)
|
||||
if tidx < ks_part:
|
||||
bv2 = gB_part[n_idx, tidx].to(cutlass.Float32)
|
||||
for m in cutlass.range_constexpr(M):
|
||||
acc[m] = acc[m] + gA_part[m, tidx].to(cutlass.Float32) * bv2
|
||||
|
||||
# Intra-warp shuffle reduction
|
||||
for m in cutlass.range_constexpr(M):
|
||||
acc[m] = cute.arch.warp_reduction_sum(acc[m])
|
||||
|
||||
# Cross-warp reduction via shared memory
|
||||
smem_red_layout = cute.make_layout((M, num_warps), stride=(num_warps, 1))
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16)
|
||||
with cute.arch.elect_one():
|
||||
for m in cutlass.range_constexpr(M):
|
||||
sm[m, wid] = acc[m]
|
||||
|
||||
# Final reduction, fused mHC SiLU epilogue, and output.
|
||||
cute.arch.sync_threads()
|
||||
if tidx == 0:
|
||||
for m in cutlass.range_constexpr(M):
|
||||
partials = sm[m, None].load()
|
||||
total = cutlass.Float32(
|
||||
partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
# Production rounding boundary: GEMM output is materialized as
|
||||
# bf16 before _hc_silu_kernel reads it.
|
||||
x_bf16 = total.to(cutlass.BFloat16)
|
||||
if n_idx < rank:
|
||||
z = x_bf16.to(cutlass.Float32) * (1.0 / hc)
|
||||
gC[m, n_idx] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16)
|
||||
else:
|
||||
gC[m, n_idx] = x_bf16
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
@@ -0,0 +1,536 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py.
|
||||
|
||||
Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough
|
||||
elsewhere, with the production bf16 rounding boundary) into the split-K kernel,
|
||||
applied after the cluster reduction where the final fp32 result materializes.
|
||||
Output is bf16 instead of fp32.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass._mlir import ir as _ir
|
||||
from cutlass._mlir.dialects import llvm as _llvm
|
||||
from cutlass.cutlass_dsl import dsl_user_op
|
||||
from cutlass.pipeline import sm90 as pipeline
|
||||
|
||||
|
||||
# Map a local smem address to the same peer CTA DSMEM offset.
|
||||
@dsl_user_op
|
||||
def set_block_rank(smem_ptr, peer_rank, *, loc=None, ip=None):
|
||||
dsmem_ptr = cute.arch.map_dsmem_ptr(smem_ptr, peer_rank, loc=loc, ip=ip)
|
||||
return cutlass.Int32(dsmem_ptr.toint(loc=loc, ip=ip))
|
||||
|
||||
|
||||
# Plain DSMEM store; mbarrier helpers do not model this reduction.
|
||||
@dsl_user_op
|
||||
def st_shared_remote_f32(remote_addr, val, *, loc=None, ip=None):
|
||||
i32 = _ir.IntegerType.get_signless(32)
|
||||
addr_ir = remote_addr.ir_value(loc=loc, ip=ip)
|
||||
val_ir = val.ir_value(loc=loc, ip=ip)
|
||||
_llvm.inline_asm(
|
||||
i32,
|
||||
[addr_ir, val_ir],
|
||||
"st.shared::cluster.f32 [$0], $1; mov.u32 $2, 0;",
|
||||
"r,f,=r",
|
||||
has_side_effects=True, # keep the inline store ordered
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
def _sigmoid_f32(x):
|
||||
return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True)
|
||||
|
||||
|
||||
class HcDownSiluMma:
|
||||
"""BF16 GEMM kernel based on clustered split-K MMA, with the mHC SiLU
|
||||
epilogue fused into the final store.
|
||||
|
||||
Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the reduced
|
||||
fp32 accumulator to bf16 (production rounding boundary), then applies
|
||||
SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through.
|
||||
Output is bf16 [M, N].
|
||||
|
||||
See the base class docstring in
|
||||
vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py for the
|
||||
GEMM structure; only the epilogue differs. The compile key is
|
||||
``(split_k, num_stages, tile_n, rank, hc)``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ab_dtype=cutlass.BFloat16,
|
||||
acc_dtype=cutlass.Float32,
|
||||
tile_n: int = 16,
|
||||
tile_k: int = 256,
|
||||
num_stages: int = 2,
|
||||
num_dma_warps: int = 4,
|
||||
split_k: int = 8,
|
||||
rank: int = 320,
|
||||
hc: int = 4,
|
||||
):
|
||||
self.ab_dtype = ab_dtype
|
||||
self.acc_dtype = acc_dtype
|
||||
self.rank = rank
|
||||
self.hc = hc
|
||||
self.tile_m = 16
|
||||
self.tile_n = tile_n
|
||||
self.tile_k = tile_k
|
||||
self.copy_bits = 128
|
||||
self.num_stages = num_stages
|
||||
self.split_k = split_k
|
||||
self.mma_shape = (16, 8, 16) # mma.sync.aligned.m16n8k16
|
||||
self.atom_layout = (1, 1, 1) # one MMA atom per warp
|
||||
self.num_dma_warps = num_dma_warps
|
||||
self.num_mma_warps = 4
|
||||
self.num_dma_threads = self.num_dma_warps * cute.arch.WARP_SIZE
|
||||
self.num_mma_threads = self.num_mma_warps * cute.arch.WARP_SIZE
|
||||
self.num_threads = self.num_dma_threads + self.num_mma_threads
|
||||
self.num_epilogue_elems = self.tile_m * self.tile_n
|
||||
self.epilogue_elems_per_thread = self.num_epilogue_elems // self.num_mma_threads
|
||||
|
||||
def _make_smem_layout_AB(self, dtype, copy_bits, smem_tiler):
|
||||
"""Build the staged swizzled SMEM layout for A or B tiles."""
|
||||
major_size = min(smem_tiler[1], 64)
|
||||
# Match swizzle span to contiguous K bytes, capped by CuTe 3-bit swizzle.
|
||||
swizzle_bits = int(math.log2(major_size * dtype.width // copy_bits))
|
||||
swizzle_bits = min(swizzle_bits, 3)
|
||||
# Tile the swizzled atom across (M_or_N, K, stages).
|
||||
layout_atom_outer = cute.make_layout((8, major_size), stride=(major_size, 1))
|
||||
layout_atom = cute.make_composed_layout(
|
||||
cute.make_swizzle(swizzle_bits, 3, 3), 0, layout_atom_outer
|
||||
)
|
||||
return cute.tile_to_shape(layout_atom, smem_tiler, (0, 1, 2))
|
||||
|
||||
def _make_gmem_tiled_copy(self, atom_copy, dtype, copy_bits, num_threads):
|
||||
"""Build the per-thread cp.async vector-copy layout."""
|
||||
# Lay threads across K so each lane issues one vector copy.
|
||||
copy_elems = copy_bits // dtype.width
|
||||
k_threads = cute.size(self.tile_k) // copy_elems # threads along K
|
||||
thread_layout = cute.make_layout(
|
||||
(num_threads // k_threads, k_threads), stride=(k_threads, 1)
|
||||
)
|
||||
value_layout = cute.make_layout((1, copy_elems))
|
||||
return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout)
|
||||
|
||||
@cute.jit
|
||||
def _fill_pred(self, pred_flat, coord_tensor, k_tile, dim_limit, K_total):
|
||||
# Predicate one K tile and keep the stage-broadcast view in sync.
|
||||
coord_ktile = coord_tensor[None, None, 0, k_tile]
|
||||
num_vec = pred_flat.shape[0]
|
||||
num_mn = pred_flat.shape[1]
|
||||
for v in cutlass.range_constexpr(num_vec):
|
||||
# pred_flat is (K_VEC, M/N) for one K_TILE.
|
||||
for j in cutlass.range_constexpr(num_mn):
|
||||
pred_flat[v, j] = cute.elem_less(
|
||||
coord_ktile[(0, v), j], (dim_limit, K_total)
|
||||
)
|
||||
|
||||
def _make_pred(self, tXcX, k_tile, dim_limit, K_total):
|
||||
# pred_flat is (K_VEC, M/N); pred is (K_VEC, M/N, STAGE).
|
||||
num_vec = tXcX.shape[0][1]
|
||||
num_mn = cute.size(tXcX, mode=[1])
|
||||
pred_flat = cute.make_rmem_tensor(
|
||||
cute.make_layout((num_vec, num_mn), stride=(num_mn, 1)),
|
||||
cutlass.Boolean,
|
||||
)
|
||||
pred = cute.make_tensor(
|
||||
pred_flat.iterator,
|
||||
cute.make_layout(
|
||||
(num_vec, num_mn, cute.size(tXcX, mode=[2])),
|
||||
stride=(num_mn, 1, 0),
|
||||
),
|
||||
)
|
||||
return pred_flat, pred
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
mA: cute.Tensor,
|
||||
mB: cute.Tensor,
|
||||
mC: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
bM, bN, bK = self.tile_m, self.tile_n, self.tile_k
|
||||
copy_bits: cutlass.Constexpr = self.copy_bits
|
||||
sA_layout = self._make_smem_layout_AB(
|
||||
mA.element_type, copy_bits, (bM, bK, self.num_stages)
|
||||
)
|
||||
sB_layout = self._make_smem_layout_AB(
|
||||
mB.element_type, copy_bits, (bN, bK, self.num_stages)
|
||||
)
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
a: cute.struct.Align[
|
||||
cute.struct.MemRange[mA.element_type, cute.cosize(sA_layout)], 16
|
||||
]
|
||||
b: cute.struct.Align[
|
||||
cute.struct.MemRange[mB.element_type, cute.cosize(sB_layout)], 16
|
||||
]
|
||||
mbar: cute.struct.Align[
|
||||
cute.struct.MemRange[cutlass.Int64, self.num_stages * 2], 8
|
||||
]
|
||||
|
||||
atom_g2s = cute.make_copy_atom(
|
||||
cute.nvgpu.cpasync.CopyG2SOp(
|
||||
cache_mode=cute.nvgpu.cpasync.LoadCacheMode.GLOBAL
|
||||
),
|
||||
mA.element_type,
|
||||
num_bits_per_copy=copy_bits,
|
||||
) # cp.async GMEM -> SMEM, bypassing L1
|
||||
tiled_copy_A = self._make_gmem_tiled_copy(
|
||||
atom_g2s, mA.element_type, copy_bits, self.num_dma_threads
|
||||
)
|
||||
tiled_copy_B = self._make_gmem_tiled_copy(
|
||||
atom_g2s, mB.element_type, copy_bits, self.num_dma_threads
|
||||
)
|
||||
op = cute.nvgpu.warp.MmaF16BF16Op(self.ab_dtype, self.acc_dtype, self.mma_shape)
|
||||
# Repeat the m16n8k16 atom along N to cover the CTA output tile.
|
||||
perm_mnk = (
|
||||
self.atom_layout[0] * self.mma_shape[0],
|
||||
self.atom_layout[1] * self.mma_shape[1] * (self.tile_n // 8),
|
||||
self.atom_layout[2] * self.mma_shape[2],
|
||||
)
|
||||
tiled_mma = cute.make_tiled_mma(
|
||||
op, cute.make_layout(self.atom_layout), permutation_mnk=perm_mnk
|
||||
)
|
||||
tiler_mn = (bM, bN)
|
||||
grid_m, grid_n = cute.ceil_div(mC.shape, tiler_mn)
|
||||
self.kernel(
|
||||
mA,
|
||||
mB,
|
||||
mC,
|
||||
sA_layout,
|
||||
sB_layout,
|
||||
tiled_copy_A,
|
||||
tiled_copy_B,
|
||||
tiled_mma,
|
||||
SharedStorage,
|
||||
self.rank,
|
||||
self.hc,
|
||||
).launch(
|
||||
grid=[
|
||||
cute.size(grid_m),
|
||||
cute.size(grid_n),
|
||||
self.split_k,
|
||||
],
|
||||
block=[self.num_threads, 1, 1],
|
||||
cluster=[
|
||||
1,
|
||||
1,
|
||||
self.split_k,
|
||||
], # split-K CTAs form one cluster
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
mA,
|
||||
mB,
|
||||
mC,
|
||||
sA_layout: cute.ComposedLayout,
|
||||
sB_layout: cute.ComposedLayout,
|
||||
tiled_copy_A: cute.TiledCopy,
|
||||
tiled_copy_B: cute.TiledCopy,
|
||||
tiled_mma: cute.TiledMma,
|
||||
shared_storage: cutlass.Constexpr,
|
||||
rank: cutlass.Constexpr,
|
||||
hc: cutlass.Constexpr,
|
||||
):
|
||||
bM, bN, bK = self.tile_m, self.tile_n, self.tile_k
|
||||
num_stages = self.num_stages
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
bid_m, bid_n, bid_z = cute.arch.block_idx()
|
||||
warp_idx = cute.arch.warp_idx()
|
||||
lane_id = cute.arch.lane_idx()
|
||||
num_dma_warps: cutlass.Constexpr = self.num_dma_warps
|
||||
is_dma_warp = warp_idx < num_dma_warps
|
||||
dma_tidx = tidx
|
||||
mma_tidx = tidx - self.num_dma_threads
|
||||
mma_warp_idx = warp_idx - num_dma_warps
|
||||
|
||||
cta_tiler = (bM, bN, bK)
|
||||
coord = (bid_m, bid_n, None) # all K tiles
|
||||
# CTA-local tiles.
|
||||
gA = cute.local_tile(
|
||||
mA, tiler=cta_tiler, coord=coord, proj=(1, None, 1)
|
||||
) # skip N
|
||||
gB = cute.local_tile(
|
||||
mB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1)
|
||||
) # skip M
|
||||
gC = cute.local_tile(
|
||||
mC, tiler=cta_tiler, coord=coord, proj=(1, 1, None)
|
||||
) # skip K
|
||||
|
||||
mcA = cute.make_identity_tensor(mA.layout.shape)
|
||||
mcB = cute.make_identity_tensor(mB.layout.shape)
|
||||
mcC = cute.make_identity_tensor(mC.layout.shape)
|
||||
# Coordinate modes: cA=(M,K,k_tile), cB=(N,K,k_tile), cC=(M,N).
|
||||
cA = cute.local_tile(mcA, tiler=cta_tiler, coord=coord, proj=(1, None, 1))
|
||||
cB = cute.local_tile(mcB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1))
|
||||
cC = cute.local_tile(mcC, tiler=cta_tiler, coord=coord, proj=(1, 1, None))
|
||||
|
||||
# 128-bit cp.async copies require 16-byte aligned GMEM views.
|
||||
gA = cute.make_tensor(gA.iterator.align(16), gA.layout)
|
||||
gB = cute.make_tensor(gB.iterator.align(16), gB.layout)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage_ptr = smem.allocate(shared_storage.size_in_bytes(), byte_alignment=16) # type: ignore[attr-defined]
|
||||
storage = shared_storage(storage_ptr) # type: ignore[call-arg]
|
||||
sA = storage.a.get_tensor(sA_layout)
|
||||
sB = storage.b.get_tensor(sB_layout)
|
||||
|
||||
# Pipeline cp.async producers into MMA consumers.
|
||||
producer_group = pipeline.CooperativeGroup(
|
||||
pipeline.Agent.Thread, self.num_dma_threads
|
||||
)
|
||||
consumer_group = pipeline.CooperativeGroup(
|
||||
pipeline.Agent.Thread, self.num_mma_threads
|
||||
)
|
||||
mainloop_pipeline = pipeline.PipelineCpAsync.create(
|
||||
barrier_storage=storage.mbar.data_ptr(),
|
||||
num_stages=num_stages,
|
||||
producer_group=producer_group,
|
||||
consumer_group=consumer_group,
|
||||
)
|
||||
|
||||
# Round-robin split-K: split z handles tiles z, z+split_k, ...
|
||||
K_total = cute.size(mA, mode=[1])
|
||||
k_tile_count = cute.size(gA, mode=[2])
|
||||
k_start = bid_z
|
||||
num_k_tiles = cute.ceil_div(k_tile_count - k_start, self.split_k)
|
||||
|
||||
if is_dma_warp:
|
||||
# DMA warps trade registers for copy throughput.
|
||||
cute.arch.setmaxregister_decrease(40)
|
||||
thr_A = tiled_copy_A.get_slice(dma_tidx)
|
||||
thr_B = tiled_copy_B.get_slice(dma_tidx)
|
||||
tAgA = thr_A.partition_S(gA)
|
||||
tAsA = thr_A.partition_D(sA)
|
||||
tBgB = thr_B.partition_S(gB)
|
||||
tBsB = thr_B.partition_D(sB)
|
||||
tAcA = thr_A.partition_S(cA)
|
||||
tBcB = thr_B.partition_S(cB)
|
||||
|
||||
# Build M/K and N/K predicates, broadcast across K-tile copies.
|
||||
tApA_flat, tApA = self._make_pred(tAcA, k_start, mA.shape[0], K_total)
|
||||
tBpB_flat, tBpB = self._make_pred(tBcB, k_start, mB.shape[0], K_total)
|
||||
self._fill_pred(tApA_flat, tAcA, k_start, mA.shape[0], K_total)
|
||||
self._fill_pred(tBpB_flat, tBcB, k_start, mB.shape[0], K_total)
|
||||
|
||||
producer_state = pipeline.make_pipeline_state(
|
||||
pipeline.PipelineUserType.Producer, num_stages
|
||||
)
|
||||
|
||||
# Prime the first pipeline stage.
|
||||
mainloop_pipeline.producer_acquire(producer_state)
|
||||
cute.copy(
|
||||
tiled_copy_B,
|
||||
tBgB[None, None, None, k_start],
|
||||
tBsB[None, None, None, producer_state.index],
|
||||
pred=tBpB,
|
||||
)
|
||||
cute.arch.griddepcontrol_wait()
|
||||
cute.copy(
|
||||
tiled_copy_A,
|
||||
tAgA[None, None, None, k_start],
|
||||
tAsA[None, None, None, producer_state.index],
|
||||
pred=tApA,
|
||||
)
|
||||
mainloop_pipeline.producer_commit(producer_state)
|
||||
producer_state.advance()
|
||||
|
||||
for k_tile in cutlass.range(
|
||||
k_start + self.split_k, k_tile_count, self.split_k, unroll=1
|
||||
):
|
||||
self._fill_pred(tApA_flat, tAcA, k_tile, mA.shape[0], K_total)
|
||||
self._fill_pred(tBpB_flat, tBcB, k_tile, mB.shape[0], K_total)
|
||||
mainloop_pipeline.producer_acquire(producer_state)
|
||||
cute.copy(
|
||||
tiled_copy_A,
|
||||
tAgA[None, None, None, k_tile],
|
||||
tAsA[None, None, None, producer_state.index],
|
||||
pred=tApA,
|
||||
)
|
||||
cute.copy(
|
||||
tiled_copy_B,
|
||||
tBgB[None, None, None, k_tile],
|
||||
tBsB[None, None, None, producer_state.index],
|
||||
pred=tBpB,
|
||||
)
|
||||
mainloop_pipeline.producer_commit(producer_state)
|
||||
producer_state.advance()
|
||||
|
||||
mainloop_pipeline.producer_tail(producer_state)
|
||||
|
||||
else:
|
||||
# MMA warps with k-phase interleaving
|
||||
cute.arch.setmaxregister_increase(232) # large MMA fragments
|
||||
|
||||
num_mma_warps: cutlass.Constexpr = self.num_mma_warps
|
||||
|
||||
thr_mma = tiled_mma.get_slice(lane_id)
|
||||
tCsA = thr_mma.partition_A(sA)
|
||||
tCsB = thr_mma.partition_B(sB)
|
||||
tCgC = thr_mma.partition_C(gC)
|
||||
|
||||
tCrA = tiled_mma.make_fragment_A(tCsA[None, None, None, 0])
|
||||
tCrB = tiled_mma.make_fragment_B(tCsB[None, None, None, 0])
|
||||
tCrC = tiled_mma.make_fragment_C(tCgC)
|
||||
tCrC.fill(0.0)
|
||||
|
||||
# ldmatrix moves SMEM fragments into MMA registers.
|
||||
atom_s2r_A = cute.make_copy_atom(
|
||||
cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mA.element_type
|
||||
) # non-transposed, x4
|
||||
atom_s2r_B = cute.make_copy_atom(
|
||||
cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mB.element_type
|
||||
)
|
||||
|
||||
# SMEM -> register copy path.
|
||||
tiled_s2r_A = cute.make_tiled_copy_A(atom_s2r_A, tiled_mma)
|
||||
tiled_s2r_B = cute.make_tiled_copy_B(atom_s2r_B, tiled_mma)
|
||||
thr_s2r_A = tiled_s2r_A.get_slice(lane_id)
|
||||
thr_s2r_B = tiled_s2r_B.get_slice(lane_id)
|
||||
tCsA_v = thr_s2r_A.partition_S(sA) # views, not copies
|
||||
tCrA_v = thr_s2r_A.retile(tCrA)
|
||||
tCsB_v = thr_s2r_B.partition_S(sB)
|
||||
tCrB_v = thr_s2r_B.retile(tCrB)
|
||||
# Split the MMA K-fragments across the MMA warps.
|
||||
tCsA_warp_v = cute.logical_divide(tCsA_v, (None, None, num_mma_warps, None))
|
||||
tCsB_warp_v = cute.logical_divide(tCsB_v, (None, None, num_mma_warps, None))
|
||||
|
||||
num_k_blocks = cute.size(tCrA, mode=[2])
|
||||
k_blocks_per_warp: cutlass.Constexpr = num_k_blocks // num_mma_warps
|
||||
|
||||
consumer_state = pipeline.make_pipeline_state(
|
||||
pipeline.PipelineUserType.Consumer, num_stages
|
||||
)
|
||||
|
||||
# Shape-dynamic split-K count, so this stays a runtime range.
|
||||
for _ in cutlass.range(num_k_tiles, unroll_full=True):
|
||||
mainloop_pipeline.consumer_wait(consumer_state)
|
||||
for ki in cutlass.range_constexpr(k_blocks_per_warp):
|
||||
cute.copy(
|
||||
tiled_s2r_A,
|
||||
tCsA_warp_v[
|
||||
None, None, (mma_warp_idx, ki), consumer_state.index
|
||||
],
|
||||
tCrA_v[None, None, 0],
|
||||
) # ldmatrix
|
||||
cute.copy(
|
||||
tiled_s2r_B,
|
||||
tCsB_warp_v[
|
||||
None, None, (mma_warp_idx, ki), consumer_state.index
|
||||
],
|
||||
tCrB_v[None, None, 0],
|
||||
)
|
||||
cute.gemm(
|
||||
tiled_mma, tCrC, tCrA[None, None, 0], tCrB[None, None, 0], tCrC
|
||||
) # mma.sync
|
||||
mainloop_pipeline.consumer_release(consumer_state)
|
||||
consumer_state.advance()
|
||||
|
||||
# Cluster reduction epilogue.
|
||||
# Reduce per-warp accumulators within this CTA.
|
||||
num_elems: cutlass.Constexpr = self.num_epilogue_elems
|
||||
elems_per_thread: cutlass.Constexpr = self.epilogue_elems_per_thread
|
||||
# Map MMA threads to linear CTA output elements.
|
||||
epilogue_thread_layout = cute.make_layout(
|
||||
(elems_per_thread, self.num_mma_threads),
|
||||
stride=(self.num_mma_threads, 1),
|
||||
)
|
||||
epilogue_slots = cute.make_tensor(0, epilogue_thread_layout)
|
||||
epilogue_slot_coords = cute.make_identity_tensor((bN, bM))
|
||||
# Layout: (mma_warp, linear MN element).
|
||||
smem_red = cute.make_tensor(
|
||||
cute.arch.alloc_smem(
|
||||
cutlass.Float32, num_elems * num_mma_warps, alignment=16
|
||||
),
|
||||
cute.make_layout((num_mma_warps, num_elems), stride=(num_elems, 1)),
|
||||
)
|
||||
smem_warp = cute.make_tensor(
|
||||
cute.domain_offset((mma_warp_idx, 0), smem_red).iterator,
|
||||
cute.make_layout((bM, bN), stride=(bN, 1)),
|
||||
)
|
||||
tCsC_partial = thr_mma.partition_C(smem_warp)
|
||||
cute.autovec_copy(tCrC, tCsC_partial)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
# Layout: (split-K rank, linear MN element).
|
||||
partials = cute.make_tensor(
|
||||
cute.arch.alloc_smem(
|
||||
cutlass.Float32, num_elems * self.split_k, alignment=16
|
||||
),
|
||||
cute.make_layout((self.split_k, num_elems), stride=(num_elems, 1)),
|
||||
)
|
||||
cta_rank = cute.arch.block_idx_in_cluster()
|
||||
|
||||
for ei in cutlass.range_constexpr(elems_per_thread):
|
||||
elem_idx = epilogue_slots[ei, mma_tidx]
|
||||
local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0])
|
||||
total = cutlass.Float32(0.0)
|
||||
if cute.elem_less(local_coord, gC.shape):
|
||||
total = (
|
||||
smem_red[None, elem_idx]
|
||||
.load()
|
||||
.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
partials[cta_rank, elem_idx] = total
|
||||
|
||||
cute.arch.sync_threads()
|
||||
|
||||
# Broadcast this CTA's partials to peer DSMEM.
|
||||
for ei in cutlass.range_constexpr(elems_per_thread):
|
||||
elem_idx = epilogue_slots[ei, mma_tidx]
|
||||
my_slot = cute.domain_offset((cta_rank, elem_idx), partials).iterator
|
||||
my_val = partials[cta_rank, elem_idx]
|
||||
for peer in cutlass.range_constexpr(self.split_k):
|
||||
remote = set_block_rank(my_slot, cutlass.Int32(peer))
|
||||
st_shared_remote_f32(remote, my_val)
|
||||
|
||||
cute.arch.cluster_arrive()
|
||||
cute.arch.cluster_wait() # peer DSMEM stores are now visible
|
||||
|
||||
if mma_tidx == 0:
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
cute.arch.sync_threads()
|
||||
|
||||
# Reduce split-K partials, apply the fused mHC SiLU epilogue, and
|
||||
# write the bf16 global output.
|
||||
for ei in cutlass.range_constexpr(elems_per_thread):
|
||||
elem_idx = epilogue_slots[ei, mma_tidx]
|
||||
local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0])
|
||||
global_coord = cC[local_coord]
|
||||
if cute.elem_less(global_coord, mC.shape):
|
||||
acc = cutlass.Float32(
|
||||
partials[None, elem_idx]
|
||||
.load()
|
||||
.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
)
|
||||
# Production rounding boundary: the GEMM result is
|
||||
# materialized as bf16 before _hc_silu_kernel reads it.
|
||||
x_bf16 = acc.to(cutlass.BFloat16)
|
||||
if global_coord[1] < rank:
|
||||
z = x_bf16.to(cutlass.Float32) * (1.0 / hc)
|
||||
gC[local_coord] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16)
|
||||
else:
|
||||
gC[local_coord] = x_bf16
|
||||
|
||||
cute.arch.sync_threads()
|
||||
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py.
|
||||
|
||||
Fused Qwen4Exp mHC down projection + SiLU epilogue, where
|
||||
columns < lora_rank get ``silu(bf16(acc) / hc_count)``, the hc_count
|
||||
injection-logit columns pass through as ``bf16(acc)``, and the pad columns
|
||||
are never computed. Output is bf16 (ll_bf16's fp32-output bonus is given up
|
||||
to preserve the production rounding boundary).
|
||||
|
||||
The GEMM uses an FMA backend for M <= 4 and a split-K MMA backend beyond,
|
||||
both with PDL. The model uses its Linear module above M=48.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Literal
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The fused kernel stops winning past M ~ 64 on Qwen3.8-Next-Flash.
|
||||
MAX_FUSED_M = 48
|
||||
|
||||
# Split-K configs tuned on SM100f for Qwen3.8-Flash-Next (K=10240, N=324).
|
||||
_SM100F_TUNED_SPLITK: dict[int, tuple[int, int, int]] = {
|
||||
**{m: (6, 5, 8) for m in range(5, 17)},
|
||||
**{m: (6, 5, 16) for m in range(17, 33)},
|
||||
**{m: (6, 4, 16) for m in range(33, MAX_FUSED_M + 1)},
|
||||
}
|
||||
|
||||
# FMA: (backend, M, K, threadblock_size); MMA: (backend, split_k, stages, tile_n).
|
||||
CompileKey = tuple[Literal["fma", "mma"], int, int, int]
|
||||
|
||||
|
||||
class HcDownSiluGemm:
|
||||
"""Dispatch/compile cache for the fused mHC down+SiLU GEMM."""
|
||||
|
||||
def __init__(
|
||||
self, rank: int, hc: int, k: int, *, prefetch_pdl_weights: bool = False
|
||||
) -> None:
|
||||
self.rank = rank
|
||||
self.hc = hc
|
||||
self.k = k
|
||||
self._prefetch_pdl_weights = prefetch_pdl_weights
|
||||
self._compiled: dict[CompileKey, Any] = {}
|
||||
self._warmup_m: set[int] = set()
|
||||
self._warmup_registered = False
|
||||
|
||||
def dispatch(self, m: int) -> CompileKey:
|
||||
if m <= 4 or self.k < 2048:
|
||||
return ("fma", m, self.k, 128)
|
||||
return ("mma", *_SM100F_TUNED_SPLITK[m])
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key in self._compiled:
|
||||
return
|
||||
|
||||
import cutlass.cute as cute
|
||||
from cutlass import BFloat16
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
N = cute.sym_int()
|
||||
gemm: Any
|
||||
extra_args: tuple[int, ...]
|
||||
if compile_key[0] == "mma":
|
||||
from ._hc_down_silu_mma import HcDownSiluMma
|
||||
|
||||
_, split_k, num_stages, tile_n = compile_key
|
||||
M, K = cute.sym_int(), cute.sym_int()
|
||||
gemm = HcDownSiluMma(
|
||||
tile_n=tile_n,
|
||||
num_stages=num_stages,
|
||||
split_k=split_k,
|
||||
rank=self.rank,
|
||||
hc=self.hc,
|
||||
)
|
||||
extra_args = ()
|
||||
options = "--enable-tvm-ffi"
|
||||
else:
|
||||
from ._hc_down_silu_fma import HcDownSiluFma
|
||||
|
||||
_, M, K, threadblock_size = compile_key
|
||||
gemm = HcDownSiluFma(
|
||||
k=K,
|
||||
threadblock_size=threadblock_size,
|
||||
prefetch_pdl_weights=self._prefetch_pdl_weights,
|
||||
rank=self.rank,
|
||||
hc=self.hc,
|
||||
)
|
||||
extra_args = (M, K, 1) # runtime N placeholder
|
||||
options = "--enable-tvm-ffi --ptxas-options -maxrregcount=64"
|
||||
|
||||
hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=8)
|
||||
router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=8)
|
||||
output = make_fake_tensor(BFloat16, (M, N), divisibility=1)
|
||||
self._compiled[compile_key] = cute.compile(
|
||||
gemm,
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
*extra_args,
|
||||
cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True),
|
||||
options=options,
|
||||
)
|
||||
logger.debug("Compiled hc_down_silu: %s", compile_key)
|
||||
|
||||
def request_warmup(self, m_values: Iterable[int]) -> None:
|
||||
m_values = set(m_values)
|
||||
if not m_values:
|
||||
return
|
||||
self._warmup_m.update(m_values)
|
||||
if self._warmup_registered:
|
||||
return
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
|
||||
register_cutedsl_warmup_provider(self)
|
||||
self._warmup_registered = True
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self):
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit
|
||||
|
||||
return tuple(
|
||||
CuTeDSLCompileUnit(
|
||||
name="Qwen4Exp HC down+SiLU GEMM",
|
||||
key=(
|
||||
"qwen4-exp-hc-down-silu",
|
||||
self.rank,
|
||||
self.hc,
|
||||
self.k,
|
||||
self._prefetch_pdl_weights,
|
||||
compile_key,
|
||||
),
|
||||
compile=partial(self.compile, compile_key),
|
||||
)
|
||||
for compile_key in dict.fromkeys(
|
||||
self.dispatch(m) for m in sorted(self._warmup_m)
|
||||
)
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor, # [M, K] bf16
|
||||
router_weight: torch.Tensor, # [N, K] bf16
|
||||
) -> torch.Tensor: # [M, N] bf16
|
||||
M = hidden_states.shape[0]
|
||||
N = router_weight.shape[0]
|
||||
n_compute = self.rank + self.hc
|
||||
w_gemm = router_weight[:n_compute]
|
||||
compile_key = self.dispatch(M)
|
||||
self.compile(compile_key)
|
||||
kernel = self._compiled[compile_key]
|
||||
|
||||
output = torch.empty(M, N, dtype=torch.bfloat16, device=hidden_states.device)
|
||||
out_gemm = output[:, :n_compute]
|
||||
if compile_key[0] == "mma":
|
||||
kernel(hidden_states, w_gemm, out_gemm)
|
||||
else:
|
||||
kernel(hidden_states, w_gemm, out_gemm, n_compute)
|
||||
return output
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _get_kernel(
|
||||
rank: int, hc: int, k: int, prefetch_pdl_weights: bool = False
|
||||
) -> HcDownSiluGemm:
|
||||
return HcDownSiluGemm(rank, hc, k, prefetch_pdl_weights=prefetch_pdl_weights)
|
||||
|
||||
|
||||
def hc_down_silu(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
rank: int,
|
||||
hc: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused Qwen4Exp HC down projection + SiLU.
|
||||
|
||||
Args:
|
||||
x: Normalized hyper-hidden input, [M, K].
|
||||
weight: Merged down+inject weight, [N, K].
|
||||
rank: Number of low-rank output columns.
|
||||
hc: Number of injection-logit output columns.
|
||||
|
||||
Returns:
|
||||
LoRA activations [M, rank] and injection logits [M, hc].
|
||||
|
||||
"""
|
||||
kernel = _get_kernel(rank, hc, weight.shape[1], x.shape[0] == 1)
|
||||
output = kernel(x, weight)
|
||||
return output[:, :rank], output[:, rank : rank + hc]
|
||||
|
||||
|
||||
def request_hc_down_silu_warmup(
|
||||
m_values: Iterable[int], rank: int, hc: int, k: int
|
||||
) -> None:
|
||||
"""Precompile the fused kernels for CUDA-graph capture token counts.
|
||||
|
||||
Args:
|
||||
m_values: Token counts the model may capture CUDA graphs for; values
|
||||
outside the fused dispatch range are ignored.
|
||||
rank: Number of low-rank output columns.
|
||||
hc: Number of injection-logit output columns.
|
||||
k: Input feature size.
|
||||
|
||||
"""
|
||||
if not current_platform.has_device_capability(90) or k % 8 != 0:
|
||||
return
|
||||
m_set = {int(m) for m in m_values if 1 <= m <= MAX_FUSED_M}
|
||||
# M=1 dispatches to the weight-prefetching PDL variant.
|
||||
_get_kernel(rank, hc, k, True).request_warmup(m_set & {1})
|
||||
_get_kernel(rank, hc, k).request_warmup(m_set - {1})
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_FUSED_M",
|
||||
"hc_down_silu",
|
||||
"request_hc_down_silu_warmup",
|
||||
]
|
||||
Reference in New Issue
Block a user