diff --git a/tests/models/qwen4_exp/test_hc_ops.py b/tests/models/qwen4_exp/test_hc_ops.py index cb1a85760e..cdad1cec13 100644 --- a/tests/models/qwen4_exp/test_hc_ops.py +++ b/tests/models/qwen4_exp/test_hc_ops.py @@ -4,11 +4,13 @@ import pytest import torch +from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu from vllm.models.qwen4_exp.nvidia.ops.hc import ( grouped_gemma_rmsnorm, hc_combine, hc_combine_norm, hc_gate_mix, + hc_silu, ) from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON @@ -22,6 +24,13 @@ HC = 4 HIDDEN_SIZE = 2560 HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE EPS = 1e-6 +LORA_RANK = 320 +DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded + +requires_sm90 = pytest.mark.skipif( + not current_platform.has_device_capability(90), + reason="fused HC down+SiLU requires SM90+", +) def test_grouped_gemma_rmsnorm() -> None: @@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None: expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC) assert torch.equal(actual, expected) torch.testing.assert_close(actual_norm, expected_norm) + + +@requires_sm90 +@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48]) +def test_hc_down_silu_fused(num_tokens: int) -> None: + # Compare computed columns with the unfused ll_bf16 + hc_silu reference. + from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm + + torch.manual_seed(0) + x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + + lora, injection = hc_down_silu(x, weight, LORA_RANK, HC) + + down = ll_bf16_gemm(x, weight).to(torch.bfloat16) + torch.testing.assert_close( + lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01 + ) + torch.testing.assert_close( + injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01 + ) diff --git a/vllm/models/qwen4_exp/nvidia/hyperconnection.py b/vllm/models/qwen4_exp/nvidia/hyperconnection.py index acdba8c781..dcf8daa956 100644 --- a/vllm/models/qwen4_exp/nvidia/hyperconnection.py +++ b/vllm/models/qwen4_exp/nvidia/hyperconnection.py @@ -25,16 +25,19 @@ Typical usage inside a transformer decoder layer:: import torch from torch import nn +import vllm.envs as envs from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, ReplicatedLinear, ) from vllm.model_executor.models.utils import maybe_prefix +from vllm.platforms import current_platform from ..common.hyperconnection import ( GroupedGemmaRMSNorm, HyperConnectionConfig, ) +from .ops.cute_dsl.hc_down_silu import MAX_FUSED_M, hc_down_silu from .ops.hc import ( grouped_gemma_rmsnorm, hc_combine, @@ -58,9 +61,9 @@ class GatedResidual(nn.Module): injection. Weights: the norm owns the grouped GemmaRMSNorm affine; the projections - are vLLM Linear modules (merged replicated linear for down+inject), so - GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the - standard quant_method mechanism. + are vLLM Linear modules (merged replicated linear for down+inject). + Eligible down+inject projections use the fused SiLU GEMM; other projections + use their Linear module's GEMM dispatch. """ def __init__( @@ -105,6 +108,12 @@ class GatedResidual(nn.Module): return_bias=False, disable_tp=True, ) + weight = self.input_mix_weight_down_block_inject.weight + self._use_hc_down_silu = ( + weight.shape[1] % 8 == 0 + and weight.dtype == torch.bfloat16 + and current_platform.has_device_capability(90) + ) else: self.input_mix_weight_down = ReplicatedLinear( self.hyper_hidden_size, @@ -125,6 +134,30 @@ class GatedResidual(nn.Module): return_bias=False, ) + def _down_and_inject( + self, xn: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Down projection + SiLU; also returns injection logits if combined.""" + if not self.use_combine: + return hc_silu(self.input_mix_weight_down(xn), self.hc_count), None + + use_fused = ( + self._use_hc_down_silu + and not envs.VLLM_BATCH_INVARIANT + and 1 <= xn.shape[0] <= MAX_FUSED_M + ) + if use_fused: + return hc_down_silu( + xn, + self.input_mix_weight_down_block_inject.weight, + self.lora_rank, + self.hc_count, + ) + down_and_injection = self.input_mix_weight_down_block_inject(xn) + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + return hc_silu(lora, self.hc_count), injection + def mix( self, hidden_states: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: @@ -135,16 +168,7 @@ class GatedResidual(nn.Module): self.hc_count, ) - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) + lora, injection = self._down_and_inject(xn) gate = self.input_mix_weight_up(lora) # [M, D] block_input = hc_gate_mix(xn, gate, self.hc_count) @@ -172,16 +196,7 @@ class GatedResidual(nn.Module): self.hc_count, ) - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) + lora, injection = self._down_and_inject(xn) gate = self.input_mix_weight_up(lora) # [M, D] block_input = hc_gate_mix(xn, gate, self.hc_count) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 2e38339655..a7e24bcd2a 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -78,6 +78,7 @@ from vllm.v1.kv_cache_interface import MambaSpec from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup from .ple_layer import Qwen4ExpPLELayer from .qsa import Qwen4ExpQSAAttention @@ -686,6 +687,16 @@ class Qwen4ExpForCausalLM( ) self.set_moe_parameters(self.model.layers) enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype) + if self.model_config.dtype == torch.bfloat16: + # Precompile the fused HC down+SiLU kernels for every CUDA-graph + # capture size in the fused dispatch range, so no CuTe-DSL JIT + # happens during graph capture. + request_hc_down_silu_warmup( + vllm_config.compilation_config.cudagraph_capture_sizes or (), + self.config.hc_lowrank, + self.config.hc_count, + self.config.hidden_size * self.config.hc_count, + ) @staticmethod def get_model_state_cls(): diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py new file mode 100644 index 0000000000..d0df347566 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py @@ -0,0 +1,334 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py. + +Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough +elsewhere, with the production bf16 rounding boundary) into the FMA kernel. +Output is bf16 instead of fp32. +""" + +import cutlass +import cutlass.cute as cute +from cuda.bindings.driver import CUstream +from cutlass import const_expr + + +def _sigmoid_f32(x): + return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True) + + +class HcDownSiluFma: + """BF16 GEMM kernel based on CTA-local FMA dot products, with the mHC SiLU + epilogue fused into the final store. + + Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the fp32 + accumulator to bf16 (production rounding boundary), then applies + SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through. + Output is bf16 [M, N]. + + See the base class docstring in + vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py for the + GEMM structure; only the epilogue differs. The compile key is + ``(M, K, threadblock_size, rank, hc)``. + """ + + def __init__( + self, + k: int, + threadblock_size: int = 128, + main_vec_width: int = 8, + tail_vec_width: int = 4, + prefetch_pdl_weights: bool = False, + rank: int = 320, + hc: int = 4, + ): + self.threadblock_size = threadblock_size + self.main_vec_width = main_vec_width + self.tail_vec_width = tail_vec_width + self.prefetch_pdl_weights = prefetch_pdl_weights + self.rank = rank + self.hc = hc + self.num_warps = threadblock_size // cute.arch.WARP_SIZE + self._init_k_tiles(k) + self.main_prefetch_tiles = min(self.main_tiles, 8) + + def _vectorized_elems(self, k_extent: int, vec_width: int) -> int: + vector_tile = vec_width * self.threadblock_size + return (k_extent // vector_tile) * vector_tile + + def _init_k_tiles(self, k: int) -> None: + """Split K into vector loops, scalar rounds, and ragged tail.""" + self.k_main_elems = self._vectorized_elems(k, self.main_vec_width) + self.k_after_main = k - self.k_main_elems + self.k_tail_elems = self._vectorized_elems( + self.k_after_main, self.tail_vec_width + ) + self.k_done_all = self.k_main_elems + self.k_tail_elems + self.scalar_rem = k - self.k_done_all + self.ks_full = self.scalar_rem // self.threadblock_size + self.ks_part = self.scalar_rem % self.threadblock_size + self.k_scalar_full = self.ks_full * self.threadblock_size + self.k_part_offset = self.k_done_all + self.k_scalar_full + self.main_tiles = self.k_main_elems // ( + self.main_vec_width * self.threadblock_size + ) + self.tail_tiles = self.k_tail_elems // ( + self.tail_vec_width * self.threadblock_size + ) + + @cute.jit + def _vector_fma( + self, + acc: cute.Tensor, + tA: cute.Tensor, + tB: cute.Tensor, + M: cutlass.Constexpr, + num_tiles: cutlass.Constexpr, + align_bytes: cutlass.Constexpr, + ): + for tile in cutlass.range_constexpr(num_tiles): + bt = tB[None, tile] + br = cute.make_rmem_tensor_like(bt) + cute.autovec_copy(bt, br) + br_f32 = br.load().to(cutlass.Float32) + + for m in cutlass.range_constexpr(M): + at = tA[m, None, tile] + ar = cute.make_rmem_tensor_like(at) + cute.autovec_copy(at, ar) + vec_width: cutlass.Constexpr = cute.size(ar) + for v in cutlass.range_constexpr(vec_width): + acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v] + + @cute.jit + def _vector_fma_prefetched( + self, + acc: cute.Tensor, + tA: cute.Tensor, + tB: cute.Tensor, + rB: cute.Tensor, + M: cutlass.Constexpr, + num_tiles: cutlass.Constexpr, + prefetch_tiles: cutlass.Constexpr, + ): + for tile in cutlass.range_constexpr(num_tiles): + if const_expr(tile < prefetch_tiles): + br_f32 = rB[tile, None].load().to(cutlass.Float32) + else: + bt = tB[None, tile] + br = cute.make_rmem_tensor_like(bt) + cute.autovec_copy(bt, br) + br_f32 = br.load().to(cutlass.Float32) + for m in cutlass.range_constexpr(M): + at = tA[m, None, tile] + ar = cute.make_rmem_tensor_like(at) + cute.autovec_copy(at, ar) + vec_width: cutlass.Constexpr = cute.size(ar) + for v in cutlass.range_constexpr(vec_width): + acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v] + + def _make_thread_vector_slice( + self, + gA_vec: cute.Tensor, + gB_vec: cute.Tensor, + tidx: cutlass.Int32, + n_idx: cutlass.Int32, + threadblock_size: cutlass.Constexpr, + ): + # (M/N, K_TILE, K_LANE, K_VEC); tidx selects K_LANE. + tA = cute.logical_divide(gA_vec, (None, (None, threadblock_size))) + tB = cute.logical_divide(gB_vec, (None, (None, threadblock_size))) + return tA[None, (None, (tidx, None))], tB[n_idx, (None, (tidx, None))] + + def _make_k_slice( + self, + gX: cute.Tensor, + k_offset: cutlass.Constexpr, + k_extent: cutlass.Constexpr, + ): + k_layout_extent: cutlass.Constexpr = ( + 1 if const_expr(k_extent == 0) else k_extent + ) + + if const_expr(k_offset == 0): + return cute.local_tile( + gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0) + ) + return cute.local_tile( + cute.domain_offset((0, k_offset), gX), + (cute.size(gX, mode=[0]), k_layout_extent), + (0, 0), + ) + + @cute.jit + def __call__( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gC: cute.Tensor, + M: cutlass.Constexpr, + K_dim: cutlass.Constexpr, + N_dim: cutlass.Int32, + stream: CUstream, + ): + self.kernel( + gA, + gB, + gC, + M, + self.main_vec_width, + self.tail_vec_width, + self.threadblock_size, + self.num_warps, + self.k_main_elems, + self.k_tail_elems, + self.k_done_all, + self.ks_full, + self.ks_part, + self.k_scalar_full, + self.k_part_offset, + self.main_tiles, + self.tail_tiles, + self.rank, + self.hc, + ).launch( + grid=[N_dim, 1, 1], + block=[self.threadblock_size, 1, 1], + smem=M * 4 * self.num_warps, + stream=stream, + use_pdl=True, + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gC: cute.Tensor, + M: cutlass.Constexpr, + main_vec_width: cutlass.Constexpr, + tail_vec_width: cutlass.Constexpr, + threadblock_size: cutlass.Constexpr, + num_warps: cutlass.Constexpr, + k_main_elems: cutlass.Constexpr, + k_tail_elems: cutlass.Constexpr, + k_done_all: cutlass.Constexpr, + ks_full: cutlass.Constexpr, + ks_part: cutlass.Constexpr, + k_scalar_full: cutlass.Constexpr, + k_part_offset: cutlass.Constexpr, + main_tiles: cutlass.Constexpr, + tail_tiles: cutlass.Constexpr, + rank: cutlass.Constexpr, + hc: cutlass.Constexpr, + ): + tidx, _, _ = cute.arch.thread_idx() + n_idx, _, _ = cute.arch.block_idx() + wid = cute.arch.warp_idx() + + # One FP32 accumulator per token. + acc = cute.make_rmem_tensor((M,), cutlass.Float32) + acc.fill(0.0) + + if const_expr(k_main_elems > 0): + gA_main = self._make_k_slice(gA, 0, k_main_elems) + gB_main = self._make_k_slice(gB, 0, k_main_elems) + gA_vec = cute.logical_divide(gA_main, (None, main_vec_width)) + gB_vec = cute.logical_divide(gB_main, (None, main_vec_width)) + tA, tB = self._make_thread_vector_slice( + gA_vec, gB_vec, tidx, n_idx, threadblock_size + ) + if const_expr(self.prefetch_pdl_weights): + prefetch_tiles: cutlass.Constexpr = self.main_prefetch_tiles + prefetched_b = cute.make_rmem_tensor( + cute.make_layout( + (prefetch_tiles, main_vec_width), + stride=(main_vec_width, 1), + ), + cutlass.BFloat16, + ) + for tile in cutlass.range_constexpr(prefetch_tiles): + cute.autovec_copy(tB[None, tile], prefetched_b[tile, None]) + + cute.arch.griddepcontrol_wait() + + # 128-bit vectorized main loop + if const_expr(k_main_elems > 0): + if const_expr(self.prefetch_pdl_weights): + self._vector_fma_prefetched( + acc, + tA, + tB, + prefetched_b, + M, + main_tiles, + prefetch_tiles, + ) + else: + self._vector_fma(acc, tA, tB, M, main_tiles, 16) + + # 64-bit vectorized tail (K remainder after main loop) + if const_expr(k_tail_elems > 0): + gA_tail = self._make_k_slice(gA, k_main_elems, k_tail_elems) + gB_tail = self._make_k_slice(gB, k_main_elems, k_tail_elems) + gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width)) + gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width)) + tA_t, tB_t = self._make_thread_vector_slice( + gA_tail_vec, gB_tail_vec, tidx, n_idx, threadblock_size + ) + self._vector_fma(acc, tA_t, tB_t, M, tail_tiles, 8) + + # Full scalar rounds use CuTe width-1 tiles; KS_PART is the ragged tail. + if const_expr(ks_full > 0): + gA_scalar = self._make_k_slice(gA, k_done_all, k_scalar_full) + gB_scalar = self._make_k_slice(gB, k_done_all, k_scalar_full) + gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1)) + gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1)) + tA_s, tB_s = self._make_thread_vector_slice( + gA_scalar_vec, gB_scalar_vec, tidx, n_idx, threadblock_size + ) + self._vector_fma(acc, tA_s, tB_s, M, ks_full, 2) + + # Only threads below KS_PART load the ragged tail. + if const_expr(ks_part > 0): + gA_part = self._make_k_slice(gA, k_part_offset, ks_part) + gB_part = self._make_k_slice(gB, k_part_offset, ks_part) + if tidx < ks_part: + bv2 = gB_part[n_idx, tidx].to(cutlass.Float32) + for m in cutlass.range_constexpr(M): + acc[m] = acc[m] + gA_part[m, tidx].to(cutlass.Float32) * bv2 + + # Intra-warp shuffle reduction + for m in cutlass.range_constexpr(M): + acc[m] = cute.arch.warp_reduction_sum(acc[m]) + + # Cross-warp reduction via shared memory + smem_red_layout = cute.make_layout((M, num_warps), stride=(num_warps, 1)) + smem = cutlass.utils.SmemAllocator() + sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16) + with cute.arch.elect_one(): + for m in cutlass.range_constexpr(M): + sm[m, wid] = acc[m] + + # Final reduction, fused mHC SiLU epilogue, and output. + cute.arch.sync_threads() + if tidx == 0: + for m in cutlass.range_constexpr(M): + partials = sm[m, None].load() + total = cutlass.Float32( + partials.reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + # Production rounding boundary: GEMM output is materialized as + # bf16 before _hc_silu_kernel reads it. + x_bf16 = total.to(cutlass.BFloat16) + if n_idx < rank: + z = x_bf16.to(cutlass.Float32) * (1.0 / hc) + gC[m, n_idx] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16) + else: + gC[m, n_idx] = x_bf16 + cute.arch.griddepcontrol_launch_dependents() diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py new file mode 100644 index 0000000000..57a3aab145 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py @@ -0,0 +1,536 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py. + +Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough +elsewhere, with the production bf16 rounding boundary) into the split-K kernel, +applied after the cluster reduction where the final fp32 result materializes. +Output is bf16 instead of fp32. +""" + +import math + +import cutlass +import cutlass.cute as cute +from cuda.bindings.driver import CUstream +from cutlass._mlir import ir as _ir +from cutlass._mlir.dialects import llvm as _llvm +from cutlass.cutlass_dsl import dsl_user_op +from cutlass.pipeline import sm90 as pipeline + + +# Map a local smem address to the same peer CTA DSMEM offset. +@dsl_user_op +def set_block_rank(smem_ptr, peer_rank, *, loc=None, ip=None): + dsmem_ptr = cute.arch.map_dsmem_ptr(smem_ptr, peer_rank, loc=loc, ip=ip) + return cutlass.Int32(dsmem_ptr.toint(loc=loc, ip=ip)) + + +# Plain DSMEM store; mbarrier helpers do not model this reduction. +@dsl_user_op +def st_shared_remote_f32(remote_addr, val, *, loc=None, ip=None): + i32 = _ir.IntegerType.get_signless(32) + addr_ir = remote_addr.ir_value(loc=loc, ip=ip) + val_ir = val.ir_value(loc=loc, ip=ip) + _llvm.inline_asm( + i32, + [addr_ir, val_ir], + "st.shared::cluster.f32 [$0], $1; mov.u32 $2, 0;", + "r,f,=r", + has_side_effects=True, # keep the inline store ordered + loc=loc, + ip=ip, + ) + + +def _sigmoid_f32(x): + return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True) + + +class HcDownSiluMma: + """BF16 GEMM kernel based on clustered split-K MMA, with the mHC SiLU + epilogue fused into the final store. + + Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the reduced + fp32 accumulator to bf16 (production rounding boundary), then applies + SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through. + Output is bf16 [M, N]. + + See the base class docstring in + vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py for the + GEMM structure; only the epilogue differs. The compile key is + ``(split_k, num_stages, tile_n, rank, hc)``. + """ + + def __init__( + self, + ab_dtype=cutlass.BFloat16, + acc_dtype=cutlass.Float32, + tile_n: int = 16, + tile_k: int = 256, + num_stages: int = 2, + num_dma_warps: int = 4, + split_k: int = 8, + rank: int = 320, + hc: int = 4, + ): + self.ab_dtype = ab_dtype + self.acc_dtype = acc_dtype + self.rank = rank + self.hc = hc + self.tile_m = 16 + self.tile_n = tile_n + self.tile_k = tile_k + self.copy_bits = 128 + self.num_stages = num_stages + self.split_k = split_k + self.mma_shape = (16, 8, 16) # mma.sync.aligned.m16n8k16 + self.atom_layout = (1, 1, 1) # one MMA atom per warp + self.num_dma_warps = num_dma_warps + self.num_mma_warps = 4 + self.num_dma_threads = self.num_dma_warps * cute.arch.WARP_SIZE + self.num_mma_threads = self.num_mma_warps * cute.arch.WARP_SIZE + self.num_threads = self.num_dma_threads + self.num_mma_threads + self.num_epilogue_elems = self.tile_m * self.tile_n + self.epilogue_elems_per_thread = self.num_epilogue_elems // self.num_mma_threads + + def _make_smem_layout_AB(self, dtype, copy_bits, smem_tiler): + """Build the staged swizzled SMEM layout for A or B tiles.""" + major_size = min(smem_tiler[1], 64) + # Match swizzle span to contiguous K bytes, capped by CuTe 3-bit swizzle. + swizzle_bits = int(math.log2(major_size * dtype.width // copy_bits)) + swizzle_bits = min(swizzle_bits, 3) + # Tile the swizzled atom across (M_or_N, K, stages). + layout_atom_outer = cute.make_layout((8, major_size), stride=(major_size, 1)) + layout_atom = cute.make_composed_layout( + cute.make_swizzle(swizzle_bits, 3, 3), 0, layout_atom_outer + ) + return cute.tile_to_shape(layout_atom, smem_tiler, (0, 1, 2)) + + def _make_gmem_tiled_copy(self, atom_copy, dtype, copy_bits, num_threads): + """Build the per-thread cp.async vector-copy layout.""" + # Lay threads across K so each lane issues one vector copy. + copy_elems = copy_bits // dtype.width + k_threads = cute.size(self.tile_k) // copy_elems # threads along K + thread_layout = cute.make_layout( + (num_threads // k_threads, k_threads), stride=(k_threads, 1) + ) + value_layout = cute.make_layout((1, copy_elems)) + return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout) + + @cute.jit + def _fill_pred(self, pred_flat, coord_tensor, k_tile, dim_limit, K_total): + # Predicate one K tile and keep the stage-broadcast view in sync. + coord_ktile = coord_tensor[None, None, 0, k_tile] + num_vec = pred_flat.shape[0] + num_mn = pred_flat.shape[1] + for v in cutlass.range_constexpr(num_vec): + # pred_flat is (K_VEC, M/N) for one K_TILE. + for j in cutlass.range_constexpr(num_mn): + pred_flat[v, j] = cute.elem_less( + coord_ktile[(0, v), j], (dim_limit, K_total) + ) + + def _make_pred(self, tXcX, k_tile, dim_limit, K_total): + # pred_flat is (K_VEC, M/N); pred is (K_VEC, M/N, STAGE). + num_vec = tXcX.shape[0][1] + num_mn = cute.size(tXcX, mode=[1]) + pred_flat = cute.make_rmem_tensor( + cute.make_layout((num_vec, num_mn), stride=(num_mn, 1)), + cutlass.Boolean, + ) + pred = cute.make_tensor( + pred_flat.iterator, + cute.make_layout( + (num_vec, num_mn, cute.size(tXcX, mode=[2])), + stride=(num_mn, 1, 0), + ), + ) + return pred_flat, pred + + @cute.jit + def __call__( + self, + mA: cute.Tensor, + mB: cute.Tensor, + mC: cute.Tensor, + stream: CUstream, + ): + bM, bN, bK = self.tile_m, self.tile_n, self.tile_k + copy_bits: cutlass.Constexpr = self.copy_bits + sA_layout = self._make_smem_layout_AB( + mA.element_type, copy_bits, (bM, bK, self.num_stages) + ) + sB_layout = self._make_smem_layout_AB( + mB.element_type, copy_bits, (bN, bK, self.num_stages) + ) + + @cute.struct + class SharedStorage: + a: cute.struct.Align[ + cute.struct.MemRange[mA.element_type, cute.cosize(sA_layout)], 16 + ] + b: cute.struct.Align[ + cute.struct.MemRange[mB.element_type, cute.cosize(sB_layout)], 16 + ] + mbar: cute.struct.Align[ + cute.struct.MemRange[cutlass.Int64, self.num_stages * 2], 8 + ] + + atom_g2s = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp( + cache_mode=cute.nvgpu.cpasync.LoadCacheMode.GLOBAL + ), + mA.element_type, + num_bits_per_copy=copy_bits, + ) # cp.async GMEM -> SMEM, bypassing L1 + tiled_copy_A = self._make_gmem_tiled_copy( + atom_g2s, mA.element_type, copy_bits, self.num_dma_threads + ) + tiled_copy_B = self._make_gmem_tiled_copy( + atom_g2s, mB.element_type, copy_bits, self.num_dma_threads + ) + op = cute.nvgpu.warp.MmaF16BF16Op(self.ab_dtype, self.acc_dtype, self.mma_shape) + # Repeat the m16n8k16 atom along N to cover the CTA output tile. + perm_mnk = ( + self.atom_layout[0] * self.mma_shape[0], + self.atom_layout[1] * self.mma_shape[1] * (self.tile_n // 8), + self.atom_layout[2] * self.mma_shape[2], + ) + tiled_mma = cute.make_tiled_mma( + op, cute.make_layout(self.atom_layout), permutation_mnk=perm_mnk + ) + tiler_mn = (bM, bN) + grid_m, grid_n = cute.ceil_div(mC.shape, tiler_mn) + self.kernel( + mA, + mB, + mC, + sA_layout, + sB_layout, + tiled_copy_A, + tiled_copy_B, + tiled_mma, + SharedStorage, + self.rank, + self.hc, + ).launch( + grid=[ + cute.size(grid_m), + cute.size(grid_n), + self.split_k, + ], + block=[self.num_threads, 1, 1], + cluster=[ + 1, + 1, + self.split_k, + ], # split-K CTAs form one cluster + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def kernel( + self, + mA, + mB, + mC, + sA_layout: cute.ComposedLayout, + sB_layout: cute.ComposedLayout, + tiled_copy_A: cute.TiledCopy, + tiled_copy_B: cute.TiledCopy, + tiled_mma: cute.TiledMma, + shared_storage: cutlass.Constexpr, + rank: cutlass.Constexpr, + hc: cutlass.Constexpr, + ): + bM, bN, bK = self.tile_m, self.tile_n, self.tile_k + num_stages = self.num_stages + tidx, _, _ = cute.arch.thread_idx() + bid_m, bid_n, bid_z = cute.arch.block_idx() + warp_idx = cute.arch.warp_idx() + lane_id = cute.arch.lane_idx() + num_dma_warps: cutlass.Constexpr = self.num_dma_warps + is_dma_warp = warp_idx < num_dma_warps + dma_tidx = tidx + mma_tidx = tidx - self.num_dma_threads + mma_warp_idx = warp_idx - num_dma_warps + + cta_tiler = (bM, bN, bK) + coord = (bid_m, bid_n, None) # all K tiles + # CTA-local tiles. + gA = cute.local_tile( + mA, tiler=cta_tiler, coord=coord, proj=(1, None, 1) + ) # skip N + gB = cute.local_tile( + mB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1) + ) # skip M + gC = cute.local_tile( + mC, tiler=cta_tiler, coord=coord, proj=(1, 1, None) + ) # skip K + + mcA = cute.make_identity_tensor(mA.layout.shape) + mcB = cute.make_identity_tensor(mB.layout.shape) + mcC = cute.make_identity_tensor(mC.layout.shape) + # Coordinate modes: cA=(M,K,k_tile), cB=(N,K,k_tile), cC=(M,N). + cA = cute.local_tile(mcA, tiler=cta_tiler, coord=coord, proj=(1, None, 1)) + cB = cute.local_tile(mcB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1)) + cC = cute.local_tile(mcC, tiler=cta_tiler, coord=coord, proj=(1, 1, None)) + + # 128-bit cp.async copies require 16-byte aligned GMEM views. + gA = cute.make_tensor(gA.iterator.align(16), gA.layout) + gB = cute.make_tensor(gB.iterator.align(16), gB.layout) + + smem = cutlass.utils.SmemAllocator() + storage_ptr = smem.allocate(shared_storage.size_in_bytes(), byte_alignment=16) # type: ignore[attr-defined] + storage = shared_storage(storage_ptr) # type: ignore[call-arg] + sA = storage.a.get_tensor(sA_layout) + sB = storage.b.get_tensor(sB_layout) + + # Pipeline cp.async producers into MMA consumers. + producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_dma_threads + ) + consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_mma_threads + ) + mainloop_pipeline = pipeline.PipelineCpAsync.create( + barrier_storage=storage.mbar.data_ptr(), + num_stages=num_stages, + producer_group=producer_group, + consumer_group=consumer_group, + ) + + # Round-robin split-K: split z handles tiles z, z+split_k, ... + K_total = cute.size(mA, mode=[1]) + k_tile_count = cute.size(gA, mode=[2]) + k_start = bid_z + num_k_tiles = cute.ceil_div(k_tile_count - k_start, self.split_k) + + if is_dma_warp: + # DMA warps trade registers for copy throughput. + cute.arch.setmaxregister_decrease(40) + thr_A = tiled_copy_A.get_slice(dma_tidx) + thr_B = tiled_copy_B.get_slice(dma_tidx) + tAgA = thr_A.partition_S(gA) + tAsA = thr_A.partition_D(sA) + tBgB = thr_B.partition_S(gB) + tBsB = thr_B.partition_D(sB) + tAcA = thr_A.partition_S(cA) + tBcB = thr_B.partition_S(cB) + + # Build M/K and N/K predicates, broadcast across K-tile copies. + tApA_flat, tApA = self._make_pred(tAcA, k_start, mA.shape[0], K_total) + tBpB_flat, tBpB = self._make_pred(tBcB, k_start, mB.shape[0], K_total) + self._fill_pred(tApA_flat, tAcA, k_start, mA.shape[0], K_total) + self._fill_pred(tBpB_flat, tBcB, k_start, mB.shape[0], K_total) + + producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, num_stages + ) + + # Prime the first pipeline stage. + mainloop_pipeline.producer_acquire(producer_state) + cute.copy( + tiled_copy_B, + tBgB[None, None, None, k_start], + tBsB[None, None, None, producer_state.index], + pred=tBpB, + ) + cute.arch.griddepcontrol_wait() + cute.copy( + tiled_copy_A, + tAgA[None, None, None, k_start], + tAsA[None, None, None, producer_state.index], + pred=tApA, + ) + mainloop_pipeline.producer_commit(producer_state) + producer_state.advance() + + for k_tile in cutlass.range( + k_start + self.split_k, k_tile_count, self.split_k, unroll=1 + ): + self._fill_pred(tApA_flat, tAcA, k_tile, mA.shape[0], K_total) + self._fill_pred(tBpB_flat, tBcB, k_tile, mB.shape[0], K_total) + mainloop_pipeline.producer_acquire(producer_state) + cute.copy( + tiled_copy_A, + tAgA[None, None, None, k_tile], + tAsA[None, None, None, producer_state.index], + pred=tApA, + ) + cute.copy( + tiled_copy_B, + tBgB[None, None, None, k_tile], + tBsB[None, None, None, producer_state.index], + pred=tBpB, + ) + mainloop_pipeline.producer_commit(producer_state) + producer_state.advance() + + mainloop_pipeline.producer_tail(producer_state) + + else: + # MMA warps with k-phase interleaving + cute.arch.setmaxregister_increase(232) # large MMA fragments + + num_mma_warps: cutlass.Constexpr = self.num_mma_warps + + thr_mma = tiled_mma.get_slice(lane_id) + tCsA = thr_mma.partition_A(sA) + tCsB = thr_mma.partition_B(sB) + tCgC = thr_mma.partition_C(gC) + + tCrA = tiled_mma.make_fragment_A(tCsA[None, None, None, 0]) + tCrB = tiled_mma.make_fragment_B(tCsB[None, None, None, 0]) + tCrC = tiled_mma.make_fragment_C(tCgC) + tCrC.fill(0.0) + + # ldmatrix moves SMEM fragments into MMA registers. + atom_s2r_A = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mA.element_type + ) # non-transposed, x4 + atom_s2r_B = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mB.element_type + ) + + # SMEM -> register copy path. + tiled_s2r_A = cute.make_tiled_copy_A(atom_s2r_A, tiled_mma) + tiled_s2r_B = cute.make_tiled_copy_B(atom_s2r_B, tiled_mma) + thr_s2r_A = tiled_s2r_A.get_slice(lane_id) + thr_s2r_B = tiled_s2r_B.get_slice(lane_id) + tCsA_v = thr_s2r_A.partition_S(sA) # views, not copies + tCrA_v = thr_s2r_A.retile(tCrA) + tCsB_v = thr_s2r_B.partition_S(sB) + tCrB_v = thr_s2r_B.retile(tCrB) + # Split the MMA K-fragments across the MMA warps. + tCsA_warp_v = cute.logical_divide(tCsA_v, (None, None, num_mma_warps, None)) + tCsB_warp_v = cute.logical_divide(tCsB_v, (None, None, num_mma_warps, None)) + + num_k_blocks = cute.size(tCrA, mode=[2]) + k_blocks_per_warp: cutlass.Constexpr = num_k_blocks // num_mma_warps + + consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, num_stages + ) + + # Shape-dynamic split-K count, so this stays a runtime range. + for _ in cutlass.range(num_k_tiles, unroll_full=True): + mainloop_pipeline.consumer_wait(consumer_state) + for ki in cutlass.range_constexpr(k_blocks_per_warp): + cute.copy( + tiled_s2r_A, + tCsA_warp_v[ + None, None, (mma_warp_idx, ki), consumer_state.index + ], + tCrA_v[None, None, 0], + ) # ldmatrix + cute.copy( + tiled_s2r_B, + tCsB_warp_v[ + None, None, (mma_warp_idx, ki), consumer_state.index + ], + tCrB_v[None, None, 0], + ) + cute.gemm( + tiled_mma, tCrC, tCrA[None, None, 0], tCrB[None, None, 0], tCrC + ) # mma.sync + mainloop_pipeline.consumer_release(consumer_state) + consumer_state.advance() + + # Cluster reduction epilogue. + # Reduce per-warp accumulators within this CTA. + num_elems: cutlass.Constexpr = self.num_epilogue_elems + elems_per_thread: cutlass.Constexpr = self.epilogue_elems_per_thread + # Map MMA threads to linear CTA output elements. + epilogue_thread_layout = cute.make_layout( + (elems_per_thread, self.num_mma_threads), + stride=(self.num_mma_threads, 1), + ) + epilogue_slots = cute.make_tensor(0, epilogue_thread_layout) + epilogue_slot_coords = cute.make_identity_tensor((bN, bM)) + # Layout: (mma_warp, linear MN element). + smem_red = cute.make_tensor( + cute.arch.alloc_smem( + cutlass.Float32, num_elems * num_mma_warps, alignment=16 + ), + cute.make_layout((num_mma_warps, num_elems), stride=(num_elems, 1)), + ) + smem_warp = cute.make_tensor( + cute.domain_offset((mma_warp_idx, 0), smem_red).iterator, + cute.make_layout((bM, bN), stride=(bN, 1)), + ) + tCsC_partial = thr_mma.partition_C(smem_warp) + cute.autovec_copy(tCrC, tCsC_partial) + cute.arch.sync_threads() + + # Layout: (split-K rank, linear MN element). + partials = cute.make_tensor( + cute.arch.alloc_smem( + cutlass.Float32, num_elems * self.split_k, alignment=16 + ), + cute.make_layout((self.split_k, num_elems), stride=(num_elems, 1)), + ) + cta_rank = cute.arch.block_idx_in_cluster() + + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0]) + total = cutlass.Float32(0.0) + if cute.elem_less(local_coord, gC.shape): + total = ( + smem_red[None, elem_idx] + .load() + .reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + partials[cta_rank, elem_idx] = total + + cute.arch.sync_threads() + + # Broadcast this CTA's partials to peer DSMEM. + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + my_slot = cute.domain_offset((cta_rank, elem_idx), partials).iterator + my_val = partials[cta_rank, elem_idx] + for peer in cutlass.range_constexpr(self.split_k): + remote = set_block_rank(my_slot, cutlass.Int32(peer)) + st_shared_remote_f32(remote, my_val) + + cute.arch.cluster_arrive() + cute.arch.cluster_wait() # peer DSMEM stores are now visible + + if mma_tidx == 0: + cute.arch.griddepcontrol_launch_dependents() + cute.arch.sync_threads() + + # Reduce split-K partials, apply the fused mHC SiLU epilogue, and + # write the bf16 global output. + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0]) + global_coord = cC[local_coord] + if cute.elem_less(global_coord, mC.shape): + acc = cutlass.Float32( + partials[None, elem_idx] + .load() + .reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + # Production rounding boundary: the GEMM result is + # materialized as bf16 before _hc_silu_kernel reads it. + x_bf16 = acc.to(cutlass.BFloat16) + if global_coord[1] < rank: + z = x_bf16.to(cutlass.Float32) * (1.0 / hc) + gC[local_coord] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16) + else: + gC[local_coord] = x_bf16 + + cute.arch.sync_threads() diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py new file mode 100644 index 0000000000..2495d8c09f --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py. + +Fused Qwen4Exp mHC down projection + SiLU epilogue, where +columns < lora_rank get ``silu(bf16(acc) / hc_count)``, the hc_count +injection-logit columns pass through as ``bf16(acc)``, and the pad columns +are never computed. Output is bf16 (ll_bf16's fp32-output bonus is given up +to preserve the production rounding boundary). + +The GEMM uses an FMA backend for M <= 4 and a split-K MMA backend beyond, +both with PDL. The model uses its Linear module above M=48. +""" + +import logging +from collections.abc import Iterable +from functools import lru_cache, partial +from typing import Any, Literal + +import torch + +from vllm.platforms import current_platform + +logger = logging.getLogger(__name__) + +# The fused kernel stops winning past M ~ 64 on Qwen3.8-Next-Flash. +MAX_FUSED_M = 48 + +# Split-K configs tuned on SM100f for Qwen3.8-Flash-Next (K=10240, N=324). +_SM100F_TUNED_SPLITK: dict[int, tuple[int, int, int]] = { + **{m: (6, 5, 8) for m in range(5, 17)}, + **{m: (6, 5, 16) for m in range(17, 33)}, + **{m: (6, 4, 16) for m in range(33, MAX_FUSED_M + 1)}, +} + +# FMA: (backend, M, K, threadblock_size); MMA: (backend, split_k, stages, tile_n). +CompileKey = tuple[Literal["fma", "mma"], int, int, int] + + +class HcDownSiluGemm: + """Dispatch/compile cache for the fused mHC down+SiLU GEMM.""" + + def __init__( + self, rank: int, hc: int, k: int, *, prefetch_pdl_weights: bool = False + ) -> None: + self.rank = rank + self.hc = hc + self.k = k + self._prefetch_pdl_weights = prefetch_pdl_weights + self._compiled: dict[CompileKey, Any] = {} + self._warmup_m: set[int] = set() + self._warmup_registered = False + + def dispatch(self, m: int) -> CompileKey: + if m <= 4 or self.k < 2048: + return ("fma", m, self.k, 128) + return ("mma", *_SM100F_TUNED_SPLITK[m]) + + def compile(self, compile_key: CompileKey) -> None: + if compile_key in self._compiled: + return + + import cutlass.cute as cute + from cutlass import BFloat16 + from quack.compile_utils import make_fake_tensor + + N = cute.sym_int() + gemm: Any + extra_args: tuple[int, ...] + if compile_key[0] == "mma": + from ._hc_down_silu_mma import HcDownSiluMma + + _, split_k, num_stages, tile_n = compile_key + M, K = cute.sym_int(), cute.sym_int() + gemm = HcDownSiluMma( + tile_n=tile_n, + num_stages=num_stages, + split_k=split_k, + rank=self.rank, + hc=self.hc, + ) + extra_args = () + options = "--enable-tvm-ffi" + else: + from ._hc_down_silu_fma import HcDownSiluFma + + _, M, K, threadblock_size = compile_key + gemm = HcDownSiluFma( + k=K, + threadblock_size=threadblock_size, + prefetch_pdl_weights=self._prefetch_pdl_weights, + rank=self.rank, + hc=self.hc, + ) + extra_args = (M, K, 1) # runtime N placeholder + options = "--enable-tvm-ffi --ptxas-options -maxrregcount=64" + + hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=8) + router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=8) + output = make_fake_tensor(BFloat16, (M, N), divisibility=1) + self._compiled[compile_key] = cute.compile( + gemm, + hidden_states, + router_weight, + output, + *extra_args, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options=options, + ) + logger.debug("Compiled hc_down_silu: %s", compile_key) + + def request_warmup(self, m_values: Iterable[int]) -> None: + m_values = set(m_values) + if not m_values: + return + self._warmup_m.update(m_values) + if self._warmup_registered: + return + from vllm.model_executor.warmup.cutedsl_warmup import ( + register_cutedsl_warmup_provider, + ) + + register_cutedsl_warmup_provider(self) + self._warmup_registered = True + + def get_cutedsl_warmup_compile_units(self): + from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit + + return tuple( + CuTeDSLCompileUnit( + name="Qwen4Exp HC down+SiLU GEMM", + key=( + "qwen4-exp-hc-down-silu", + self.rank, + self.hc, + self.k, + self._prefetch_pdl_weights, + compile_key, + ), + compile=partial(self.compile, compile_key), + ) + for compile_key in dict.fromkeys( + self.dispatch(m) for m in sorted(self._warmup_m) + ) + ) + + def __call__( + self, + hidden_states: torch.Tensor, # [M, K] bf16 + router_weight: torch.Tensor, # [N, K] bf16 + ) -> torch.Tensor: # [M, N] bf16 + M = hidden_states.shape[0] + N = router_weight.shape[0] + n_compute = self.rank + self.hc + w_gemm = router_weight[:n_compute] + compile_key = self.dispatch(M) + self.compile(compile_key) + kernel = self._compiled[compile_key] + + output = torch.empty(M, N, dtype=torch.bfloat16, device=hidden_states.device) + out_gemm = output[:, :n_compute] + if compile_key[0] == "mma": + kernel(hidden_states, w_gemm, out_gemm) + else: + kernel(hidden_states, w_gemm, out_gemm, n_compute) + return output + + +@lru_cache +def _get_kernel( + rank: int, hc: int, k: int, prefetch_pdl_weights: bool = False +) -> HcDownSiluGemm: + return HcDownSiluGemm(rank, hc, k, prefetch_pdl_weights=prefetch_pdl_weights) + + +def hc_down_silu( + x: torch.Tensor, + weight: torch.Tensor, + rank: int, + hc: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused Qwen4Exp HC down projection + SiLU. + + Args: + x: Normalized hyper-hidden input, [M, K]. + weight: Merged down+inject weight, [N, K]. + rank: Number of low-rank output columns. + hc: Number of injection-logit output columns. + + Returns: + LoRA activations [M, rank] and injection logits [M, hc]. + + """ + kernel = _get_kernel(rank, hc, weight.shape[1], x.shape[0] == 1) + output = kernel(x, weight) + return output[:, :rank], output[:, rank : rank + hc] + + +def request_hc_down_silu_warmup( + m_values: Iterable[int], rank: int, hc: int, k: int +) -> None: + """Precompile the fused kernels for CUDA-graph capture token counts. + + Args: + m_values: Token counts the model may capture CUDA graphs for; values + outside the fused dispatch range are ignored. + rank: Number of low-rank output columns. + hc: Number of injection-logit output columns. + k: Input feature size. + + """ + if not current_platform.has_device_capability(90) or k % 8 != 0: + return + m_set = {int(m) for m in m_values if 1 <= m <= MAX_FUSED_M} + # M=1 dispatches to the weight-prefetching PDL variant. + _get_kernel(rank, hc, k, True).request_warmup(m_set & {1}) + _get_kernel(rank, hc, k).request_warmup(m_set - {1}) + + +__all__ = [ + "MAX_FUSED_M", + "hc_down_silu", + "request_hc_down_silu_warmup", +]