[Kernel][SM100] Add a CuTeDSL fused query kernel (#49792)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
xiaozhoupy
2026-08-04 15:23:06 -07:00
committed by GitHub
co-authored by Claude
parent c687c1abb8
commit 3756bf1e2b
6 changed files with 699 additions and 1 deletions
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
@@ -773,10 +776,12 @@ def fused_q(
``(ql_nope, q_pe)`` tuple the backend expects.
"""
assert positions.ndim == 1
assert positions.dtype == torch.int64
assert q_pe.ndim == 3
assert q_pe_cos_sin_cache.ndim == 2
assert ql_nope.ndim == 3
assert ql_nope.shape[:2] == q_pe.shape[:2]
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
num_tokens = positions.shape[0]
num_q_heads = q_pe.shape[1]
@@ -794,6 +799,23 @@ def fused_q(
assert index_weights is not None
num_index_q_heads = index_q.shape[1]
index_q_head_dim = index_q.shape[2]
# fused_q is shared with the ROCm path, and the CuTeDSL module imports
# cutlass at module scope, so only reach for it on CUDA.
cutedsl_kernel: Callable[..., None] | None = None
if current_platform.is_cuda():
from vllm.models.deepseek_v32.nvidia.ops.fused_q_cutedsl import (
fused_q_cutedsl,
is_fused_q_cutedsl_supported,
)
if is_fused_q_cutedsl_supported(
q_pe,
index_q,
ql_nope,
has_indexer=has_indexer,
quantize_mqa=quantize_mqa,
):
cutedsl_kernel = fused_q_cutedsl
grid_heads = max(mqa_grid_heads, num_index_q_heads)
if quantize_mqa:
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
@@ -815,6 +837,26 @@ def fused_q(
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
if cutedsl_kernel is not None:
cutedsl_kernel(
positions,
q_pe,
q_pe_cos_sin_cache,
ql_nope,
q_scale,
mqa_q,
index_q,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
)
return index_q_fp8, index_weights_out, mqa_q
_fused_q_kernel[(3, num_tokens, grid_heads)](
positions,
q_pe,