[Kernel][SM100] Add a CuTeDSL fused query kernel (#49792)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
@@ -773,10 +776,12 @@ def fused_q(
|
||||
``(ql_nope, q_pe)`` tuple the backend expects.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert positions.dtype == torch.int64
|
||||
assert q_pe.ndim == 3
|
||||
assert q_pe_cos_sin_cache.ndim == 2
|
||||
assert ql_nope.ndim == 3
|
||||
assert ql_nope.shape[:2] == q_pe.shape[:2]
|
||||
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
|
||||
|
||||
num_tokens = positions.shape[0]
|
||||
num_q_heads = q_pe.shape[1]
|
||||
@@ -794,6 +799,23 @@ def fused_q(
|
||||
assert index_weights is not None
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
# fused_q is shared with the ROCm path, and the CuTeDSL module imports
|
||||
# cutlass at module scope, so only reach for it on CUDA.
|
||||
cutedsl_kernel: Callable[..., None] | None = None
|
||||
if current_platform.is_cuda():
|
||||
from vllm.models.deepseek_v32.nvidia.ops.fused_q_cutedsl import (
|
||||
fused_q_cutedsl,
|
||||
is_fused_q_cutedsl_supported,
|
||||
)
|
||||
|
||||
if is_fused_q_cutedsl_supported(
|
||||
q_pe,
|
||||
index_q,
|
||||
ql_nope,
|
||||
has_indexer=has_indexer,
|
||||
quantize_mqa=quantize_mqa,
|
||||
):
|
||||
cutedsl_kernel = fused_q_cutedsl
|
||||
grid_heads = max(mqa_grid_heads, num_index_q_heads)
|
||||
if quantize_mqa:
|
||||
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
|
||||
@@ -815,6 +837,26 @@ def fused_q(
|
||||
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
if cutedsl_kernel is not None:
|
||||
cutedsl_kernel(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_cos_sin_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_q,
|
||||
index_q,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
)
|
||||
return index_q_fp8, index_weights_out, mqa_q
|
||||
|
||||
_fused_q_kernel[(3, num_tokens, grid_heads)](
|
||||
positions,
|
||||
q_pe,
|
||||
|
||||
Reference in New Issue
Block a user