[DSv4] Adding TRTLLM gen attention kernel (#43827)
This commit is contained in:
@@ -576,7 +576,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
|
||||
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLASparseBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ROCM_V4_FLASHMLA_SPARSE"
|
||||
return "ROCM_FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4ROCMAiterMLASparseMetadataBuilder"]:
|
||||
|
||||
@@ -55,9 +55,6 @@ from vllm.utils.multi_stream_utils import (
|
||||
maybe_execute_in_parallel,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
get_max_prefill_buffer_size,
|
||||
@@ -73,21 +70,82 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _select_v4_sparse_impl() -> "type[DeepseekV4SparseMLAAttentionImpl]":
|
||||
"""Pick the platform-specific V4 sparse MLA impl class. Sole platform check."""
|
||||
def _resolve_dsv4_backend(vllm_config: VllmConfig | None):
|
||||
"""Return the explicitly-requested DSv4 sparse backend enum, or None."""
|
||||
if vllm_config is None:
|
||||
return None
|
||||
attn_config = getattr(vllm_config, "attention_config", None)
|
||||
return getattr(attn_config, "backend", None) if attn_config is not None else None
|
||||
|
||||
|
||||
def _select_v4_sparse_impl(
|
||||
vllm_config: VllmConfig | None = None,
|
||||
) -> "type[DeepseekV4SparseMLAAttentionImpl]":
|
||||
"""Pick the V4 sparse MLA impl class.
|
||||
|
||||
An explicit ``--attention-backend FLASHINFER_MLA_SPARSE_DSV4`` selects the
|
||||
FlashInfer TRTLLM-gen path; otherwise the platform default (FlashMLA on
|
||||
NVIDIA, ROCm Aiter on AMD) is used.
|
||||
"""
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
backend = _resolve_dsv4_backend(vllm_config)
|
||||
if backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4:
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLASparseImpl,
|
||||
)
|
||||
|
||||
logger.info_once("Using FLASHINFER_MLA_SPARSE_DSV4 backend.")
|
||||
return DeepseekV4FlashInferMLASparseImpl
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
DeepseekV4ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
logger.info_once("Using ROCM_FLASHMLA_SPARSE_DSV4 backend.")
|
||||
return DeepseekV4ROCMAiterMLASparseImpl
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import (
|
||||
DeepseekV4FlashMLASparseImpl,
|
||||
)
|
||||
|
||||
logger.info_once("Using FLASHMLA_SPARSE_DSV4 backend.")
|
||||
return DeepseekV4FlashMLASparseImpl
|
||||
|
||||
|
||||
def _resolve_dsv4_kv_cache_dtype(
|
||||
backend,
|
||||
kv_cache_dtype: str,
|
||||
cache_config: CacheConfig | None,
|
||||
) -> tuple[str, torch.dtype]:
|
||||
"""Map ``(backend, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
|
||||
|
||||
FlashInfer V4 reads a contiguous 512-wide KV row (bf16 or per-tensor FP8
|
||||
E4M3); FlashMLA V4 reads the legacy UE8M0 paged layout (uint8 /
|
||||
``fp8_ds_mla``). For FlashMLA the canonical ``fp8_ds_mla`` string is
|
||||
written back onto ``cache_config`` so the page-size specs pick the 576B
|
||||
layout.
|
||||
"""
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
if backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4:
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
return kv_cache_dtype, torch.float8_e4m3fn
|
||||
# auto / bfloat16 -> contiguous BF16 cache.
|
||||
return kv_cache_dtype, torch.bfloat16
|
||||
|
||||
# FlashMLA (and ROCm Aiter): legacy UE8M0 paged uint8 cache.
|
||||
assert kv_cache_dtype.startswith("fp8"), (
|
||||
f"DeepseekV4 FlashMLA sparse backend only supports fp8 kv-cache, "
|
||||
f"got {kv_cache_dtype}"
|
||||
)
|
||||
if kv_cache_dtype != "fp8_ds_mla":
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
|
||||
return kv_cache_dtype, torch.uint8
|
||||
|
||||
|
||||
class DeepseekV4MLA(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -194,10 +252,17 @@ class DeepseekV4MLA(nn.Module):
|
||||
self.ln_events = [torch.cuda.Event() for _ in range(4)]
|
||||
|
||||
assert cache_config is not None, "DeepseekV4 attention requires cache_config"
|
||||
# Resolve the SWA cache tensor dtype from the selected backend: FlashMLA
|
||||
# uses the legacy UE8M0 paged uint8 layout; FlashInfer uses a contiguous
|
||||
# bf16 / per-tensor fp8 row.
|
||||
backend = _resolve_dsv4_backend(vllm_config)
|
||||
_, swa_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
|
||||
backend, cache_config.cache_dtype, cache_config
|
||||
)
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
head_dim=self.head_dim,
|
||||
window_size=self.window_size,
|
||||
dtype=torch.uint8,
|
||||
dtype=swa_cache_torch_dtype,
|
||||
prefix=f"{prefix}.swa_cache",
|
||||
cache_config=cache_config,
|
||||
)
|
||||
@@ -478,25 +543,66 @@ class DeepseekV4MLA(nn.Module):
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_kv_cache = self.swa_cache_layer.kv_cache
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
# The fused insert ops require int64 position_ids; the runner's positions
|
||||
# buffer is already int64, so no cast is needed.
|
||||
assert positions.dtype == torch.int64
|
||||
cos_sin_cache = self.rotary_emb.cos_sin_cache
|
||||
cache_dtype = swa_kv_cache.dtype
|
||||
|
||||
# Horizontally fused:
|
||||
# Q side: q_head_norm (per-head RMSNorm, no weight) + GPT-J RoPE,
|
||||
# with zero-fill for the padding head slots. The kernel
|
||||
# allocates and returns the padded q tensor.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert
|
||||
# kv is unchanged; mla_attn reads kv solely via swa_kv_cache.
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
if cache_dtype == torch.uint8:
|
||||
# Legacy FlashMLA UE8M0 paged path. Horizontally fused:
|
||||
# Q side: per-head RMSNorm (no weight) + GPT-J RoPE, zero-filling
|
||||
# the padding head slots; the kernel allocates and returns
|
||||
# the padded q tensor.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.padded_heads,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
|
||||
# FlashInfer full-cache path: contiguous [num_blocks, block_size, 512]
|
||||
# cache (no Q padding). bf16 rewrites q in place; per-tensor fp8 writes a
|
||||
# separately-allocated fp8 q and quantizes the KV row.
|
||||
block_size = swa_metadata.block_size
|
||||
swa_kv_cache_3d = swa_kv_cache.view(-1, block_size, self.head_dim)
|
||||
if cache_dtype == torch.bfloat16:
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_3d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.eps,
|
||||
block_size,
|
||||
)
|
||||
return q
|
||||
|
||||
# per-tensor fp8 (torch.float8_e4m3fn)
|
||||
q_fp8 = torch.empty_like(q, dtype=torch.float8_e4m3fn)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
q_fp8,
|
||||
swa_kv_cache_3d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions.to(torch.int64),
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.padded_heads,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.mla_attn._flashinfer_fp8_kv_scale,
|
||||
self.mla_attn._flashinfer_fp8_q_scale_inv,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
block_size,
|
||||
)
|
||||
return q_fp8
|
||||
|
||||
|
||||
class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
@@ -524,7 +630,8 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.impl_cls = _select_v4_sparse_impl()
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.impl_cls = _select_v4_sparse_impl(vllm_config)
|
||||
self.backend_cls = self.impl_cls.backend_cls
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = 1
|
||||
@@ -556,34 +663,23 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.swa_cache_layer: DeepseekV4SWACache = swa_cache_layer
|
||||
|
||||
# Get vllm config for cache setup
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.max_num_batched_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
# DeepseekV4 only supports fp8 kv-cache format for now.
|
||||
|
||||
# Resolve the kv-cache dtype from the selected backend. FlashMLA uses
|
||||
# the legacy UE8M0 paged uint8 (fp8_ds_mla) layout; FlashInfer uses a
|
||||
# contiguous bf16 / per-tensor fp8 row.
|
||||
backend = _resolve_dsv4_backend(vllm_config)
|
||||
kv_cache_dtype = cache_config.cache_dtype if cache_config is not None else "fp8"
|
||||
|
||||
assert kv_cache_dtype.startswith("fp8"), (
|
||||
f"DeepseekV4 only supports fp8 kv-cache format for now, "
|
||||
f"got {kv_cache_dtype}"
|
||||
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
|
||||
backend, kv_cache_dtype, cache_config
|
||||
)
|
||||
assert issubclass(self.get_attn_backend(), FlashMLASparseBackend), (
|
||||
"Only FlashMLA Sparse Attention backend is supported for DeepseekV4 for now"
|
||||
)
|
||||
# FlashMLA Sparse Attention fp8 backend uses "fp8_ds_mla" kv-cache format
|
||||
# Automatically convert fp8 kv-cache format to "fp8_ds_mla"
|
||||
if (
|
||||
issubclass(self.get_attn_backend(), FlashMLASparseBackend)
|
||||
and kv_cache_dtype.startswith("fp8")
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
assert cache_config is not None
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
|
||||
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
# Per-impl layer buffers (e.g. FlashInfer FP8 scale buffers). No-op for
|
||||
# the FlashMLA / ROCm impls.
|
||||
self.impl_cls.init_layer_buffers(self)
|
||||
|
||||
# Register with compilation context for metadata lookup
|
||||
compilation_config = vllm_config.compilation_config
|
||||
@@ -602,14 +698,17 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.compress_ratio <= 1
|
||||
): # SWA part. Allocated separately as DeepseekV4SWACache.
|
||||
return None
|
||||
# FlashMLA uses the UE8M0 paged uint8 layout (576B aligned); FlashInfer
|
||||
# uses a contiguous bf16 / per-tensor fp8 cache with no extra alignment.
|
||||
is_flashmla = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=torch.uint8,
|
||||
dtype=torch.uint8 if is_flashmla else self.kv_cache_torch_dtype,
|
||||
compress_ratio=self.compress_ratio,
|
||||
cache_dtype_str=self.kv_cache_dtype,
|
||||
alignment=576, # NOTE: FlashMLA requires 576B alignment
|
||||
alignment=576 if is_flashmla else None, # FlashMLA needs 576B
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
@@ -15,6 +16,7 @@ from .save_partial_states import save_partial_states
|
||||
|
||||
__all__ = [
|
||||
"MXFP4_BLOCK_SIZE",
|
||||
"build_flashinfer_mixed_sparse_indices",
|
||||
"combine_topk_swa_indices",
|
||||
"compute_global_topk_indices_and_lens",
|
||||
"dequantize_and_gather_k_cache",
|
||||
|
||||
@@ -592,3 +592,308 @@ def _combine_topk_swa_indices_kernel(
|
||||
|
||||
combined_len = topk_len + swa_len
|
||||
tl.store(combined_lens_ptr + token_idx, combined_len)
|
||||
|
||||
|
||||
def build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices: torch.Tensor,
|
||||
decode_compressed_indices: torch.Tensor | None,
|
||||
decode_compressed_topk_lens: torch.Tensor | None,
|
||||
prefill_topk_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
swa_block_table: torch.Tensor,
|
||||
swa_block_size: int,
|
||||
compressed_block_table: torch.Tensor | None,
|
||||
compressed_block_size: int,
|
||||
window_size: int,
|
||||
compress_ratio: int,
|
||||
topk: int,
|
||||
decode_compressed_indices_are_local: bool = False,
|
||||
decode_is_valid_token: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
|
||||
|
||||
Produces ``sparse_indices`` of shape ``[num_tokens, window_size +
|
||||
padded_topk]`` (the first ``window_size`` columns are SWA slot ids, the rest
|
||||
are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length per
|
||||
token). Decode tokens read precomputed SWA/compressed indices; prefill tokens
|
||||
derive their SWA window from the position and translate local compressed
|
||||
indices to global slots via the block tables.
|
||||
"""
|
||||
assert decode_swa_indices.dtype == torch.int32
|
||||
assert decode_swa_indices.dim() == 2
|
||||
assert decode_swa_indices.shape[-1] == window_size
|
||||
if decode_compressed_topk_lens is not None:
|
||||
assert decode_compressed_topk_lens.dtype == torch.int32
|
||||
assert prefill_topk_indices.dtype == torch.int32
|
||||
assert prefill_topk_indices.dim() == 2
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
assert seq_lens.dtype == torch.int32
|
||||
assert token_to_req_indices.dtype == torch.int32
|
||||
assert swa_block_table.dtype == torch.int32
|
||||
|
||||
num_decode_tokens = decode_swa_indices.shape[0]
|
||||
num_prefill_tokens = prefill_topk_indices.shape[0]
|
||||
num_tokens = num_decode_tokens + num_prefill_tokens
|
||||
assert token_to_req_indices.shape[0] >= num_tokens
|
||||
if decode_compressed_topk_lens is not None:
|
||||
assert decode_compressed_topk_lens.shape[0] >= num_decode_tokens
|
||||
|
||||
decode_compressed_topk = 0
|
||||
if decode_compressed_indices is None:
|
||||
decode_compressed_indices = prefill_topk_indices
|
||||
else:
|
||||
assert decode_compressed_indices.dtype == torch.int32
|
||||
assert decode_compressed_indices.dim() == 2
|
||||
assert decode_compressed_indices.shape[0] == num_decode_tokens
|
||||
decode_compressed_topk = decode_compressed_indices.shape[-1]
|
||||
if decode_compressed_topk > 0 and decode_compressed_indices_are_local:
|
||||
assert decode_is_valid_token is not None
|
||||
assert decode_is_valid_token.dtype == torch.bool
|
||||
assert decode_is_valid_token.shape[0] >= num_decode_tokens
|
||||
else:
|
||||
decode_is_valid_token = token_to_req_indices
|
||||
|
||||
if compressed_block_table is None:
|
||||
compressed_block_table = swa_block_table
|
||||
assert compressed_block_table.dtype == torch.int32
|
||||
has_decode_compressed_lens = decode_compressed_topk_lens is not None
|
||||
if decode_compressed_topk_lens is None:
|
||||
decode_compressed_topk_lens = token_to_req_indices
|
||||
|
||||
# The FlashInfer TRTLLM-gen sparse-MLA kernels require every per-token topk
|
||||
# index row to start on a 16-byte boundary: the kernel loads the compressed
|
||||
# indices with 128-bit (16-byte) vectorized loads, so a misaligned row would
|
||||
# fault or read across rows. 16 bytes = 4 int32 indices, so round the topk
|
||||
# width (and hence the row stride, since the SWA columns are fixed-width) up
|
||||
# to a multiple of 4. The extra columns are filled with -1 (invalid) and bounded
|
||||
# by ``sparse_topk_lens``, so padding never changes the attention result.
|
||||
padded_topk = max(topk, decode_compressed_topk)
|
||||
padded_topk = (padded_topk + 3) // 4 * 4
|
||||
sparse_indices = torch.empty(
|
||||
(num_tokens, window_size + padded_topk),
|
||||
dtype=torch.int32,
|
||||
device=decode_swa_indices.device,
|
||||
)
|
||||
sparse_topk_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=decode_swa_indices.device
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return sparse_indices, sparse_topk_lens
|
||||
|
||||
window_block_size = triton.next_power_of_2(max(window_size, 1))
|
||||
topk_block_size = triton.next_power_of_2(max(padded_topk, 1))
|
||||
max_block_size = max(window_block_size, topk_block_size)
|
||||
num_warps = 4 if max_block_size >= 256 else 1
|
||||
|
||||
_build_flashinfer_mixed_sparse_indices_kernel[(num_tokens,)](
|
||||
sparse_indices,
|
||||
sparse_indices.stride(0),
|
||||
sparse_topk_lens,
|
||||
decode_swa_indices,
|
||||
decode_swa_indices.stride(0),
|
||||
decode_compressed_indices,
|
||||
decode_compressed_indices.stride(0),
|
||||
decode_compressed_topk_lens,
|
||||
decode_is_valid_token,
|
||||
prefill_topk_indices,
|
||||
prefill_topk_indices.stride(0),
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
token_to_req_indices,
|
||||
swa_block_table,
|
||||
swa_block_table.stride(0),
|
||||
swa_block_size,
|
||||
compressed_block_table,
|
||||
compressed_block_table.stride(0),
|
||||
compressed_block_size,
|
||||
NUM_DECODE_TOKENS=num_decode_tokens,
|
||||
WINDOW_SIZE=window_size,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
TOP_K=topk,
|
||||
PADDED_TOP_K=padded_topk,
|
||||
PREFILL_TOPK_STRIDE=prefill_topk_indices.shape[-1],
|
||||
DECODE_COMPRESSED_TOPK=decode_compressed_topk,
|
||||
DECODE_COMPRESSED_INDICES_ARE_LOCAL=decode_compressed_indices_are_local,
|
||||
HAS_DECODE_COMPRESSED_LENS=has_decode_compressed_lens,
|
||||
WINDOW_BLOCK_SIZE=window_block_size,
|
||||
TOPK_BLOCK_SIZE=topk_block_size,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return sparse_indices, sparse_topk_lens
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"sparse_indices_stride",
|
||||
"decode_swa_stride",
|
||||
"decode_compressed_stride",
|
||||
"prefill_topk_stride",
|
||||
"swa_block_table_stride",
|
||||
"swa_block_size",
|
||||
"compressed_block_table_stride",
|
||||
"compressed_block_size",
|
||||
"NUM_DECODE_TOKENS",
|
||||
"PREFILL_TOPK_STRIDE",
|
||||
]
|
||||
)
|
||||
def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
sparse_indices_ptr,
|
||||
sparse_indices_stride,
|
||||
sparse_topk_lens_ptr,
|
||||
decode_swa_indices_ptr,
|
||||
decode_swa_stride,
|
||||
decode_compressed_indices_ptr,
|
||||
decode_compressed_stride,
|
||||
decode_compressed_topk_lens_ptr,
|
||||
decode_is_valid_token_ptr,
|
||||
prefill_topk_indices_ptr,
|
||||
prefill_topk_stride,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
token_to_req_indices_ptr,
|
||||
swa_block_table_ptr,
|
||||
swa_block_table_stride,
|
||||
swa_block_size,
|
||||
compressed_block_table_ptr,
|
||||
compressed_block_table_stride,
|
||||
compressed_block_size,
|
||||
NUM_DECODE_TOKENS,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
TOP_K: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
PREFILL_TOPK_STRIDE,
|
||||
DECODE_COMPRESSED_TOPK: tl.constexpr,
|
||||
DECODE_COMPRESSED_INDICES_ARE_LOCAL: tl.constexpr,
|
||||
HAS_DECODE_COMPRESSED_LENS: tl.constexpr,
|
||||
WINDOW_BLOCK_SIZE: tl.constexpr,
|
||||
TOPK_BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
|
||||
if token_idx < NUM_DECODE_TOKENS:
|
||||
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||
mask = offset < WINDOW_SIZE
|
||||
values = tl.load(
|
||||
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
tl.store(
|
||||
sparse_indices_ptr + token_idx * sparse_indices_stride + offset,
|
||||
values,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
compressed_len = tl.zeros((), dtype=tl.int32)
|
||||
for i in range(0, PADDED_TOP_K, TOPK_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, TOPK_BLOCK_SIZE)
|
||||
mask = offset < PADDED_TOP_K
|
||||
values = tl.load(
|
||||
decode_compressed_indices_ptr
|
||||
+ token_idx * decode_compressed_stride
|
||||
+ offset,
|
||||
mask=offset < DECODE_COMPRESSED_TOPK,
|
||||
other=-1,
|
||||
)
|
||||
if DECODE_COMPRESSED_INDICES_ARE_LOCAL:
|
||||
token_valid = tl.load(decode_is_valid_token_ptr + token_idx)
|
||||
is_valid = values >= 0
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
block_indices = values // compressed_block_size
|
||||
block_numbers = tl.load(
|
||||
compressed_block_table_ptr
|
||||
+ req_idx * compressed_block_table_stride
|
||||
+ block_indices,
|
||||
mask=mask & is_valid,
|
||||
other=-1,
|
||||
)
|
||||
block_offsets = values % compressed_block_size
|
||||
values = block_numbers * compressed_block_size + block_offsets
|
||||
values = tl.where(is_valid, values, -1)
|
||||
compressed_len += tl.sum((is_valid & token_valid).to(tl.int32), axis=0)
|
||||
tl.store(
|
||||
sparse_indices_ptr
|
||||
+ token_idx * sparse_indices_stride
|
||||
+ WINDOW_SIZE
|
||||
+ offset,
|
||||
values,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
if DECODE_COMPRESSED_TOPK == 0:
|
||||
compressed_len = tl.zeros((), dtype=tl.int32)
|
||||
elif not DECODE_COMPRESSED_INDICES_ARE_LOCAL:
|
||||
if HAS_DECODE_COMPRESSED_LENS:
|
||||
compressed_len = tl.load(decode_compressed_topk_lens_ptr + token_idx)
|
||||
else:
|
||||
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
|
||||
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + compressed_len)
|
||||
return
|
||||
|
||||
prefill_idx = token_idx - NUM_DECODE_TOKENS
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
query_start = tl.load(query_start_loc_ptr + req_idx)
|
||||
query_end = tl.load(query_start_loc_ptr + req_idx + 1)
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + req_idx)
|
||||
start_pos = seq_len - query_len
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
swa_start_pos = pos - swa_len + 1
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
|
||||
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||
mask = offset < WINDOW_SIZE
|
||||
pos_offset = swa_start_pos + offset
|
||||
block_indices = pos_offset // swa_block_size
|
||||
block_numbers = tl.load(
|
||||
swa_block_table_ptr + req_idx * swa_block_table_stride + block_indices,
|
||||
mask=mask & (offset < swa_len),
|
||||
other=-1,
|
||||
)
|
||||
block_offsets = pos_offset % swa_block_size
|
||||
slot_ids = block_numbers * swa_block_size + block_offsets
|
||||
slot_ids = tl.where(offset < swa_len, slot_ids, -1)
|
||||
tl.store(
|
||||
sparse_indices_ptr + token_idx * sparse_indices_stride + offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
for i in range(0, PADDED_TOP_K, TOPK_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, TOPK_BLOCK_SIZE)
|
||||
mask = offset < PADDED_TOP_K
|
||||
local_idx = tl.load(
|
||||
prefill_topk_indices_ptr + prefill_idx * prefill_topk_stride + offset,
|
||||
mask=(offset < PREFILL_TOPK_STRIDE) & (offset < topk_len),
|
||||
other=-1,
|
||||
)
|
||||
is_valid = local_idx >= 0
|
||||
block_indices = local_idx // compressed_block_size
|
||||
block_numbers = tl.load(
|
||||
compressed_block_table_ptr
|
||||
+ req_idx * compressed_block_table_stride
|
||||
+ block_indices,
|
||||
mask=mask & is_valid,
|
||||
other=-1,
|
||||
)
|
||||
block_offsets = local_idx % compressed_block_size
|
||||
slot_ids = block_numbers * compressed_block_size + block_offsets
|
||||
slot_ids = tl.where((offset < topk_len) & is_valid, slot_ids, -1)
|
||||
tl.store(
|
||||
sparse_indices_ptr
|
||||
+ token_idx * sparse_indices_stride
|
||||
+ WINDOW_SIZE
|
||||
+ offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + topk_len)
|
||||
|
||||
@@ -155,13 +155,17 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# FlashMLA's UE8M0 paged layout needs 576B alignment; the FlashInfer
|
||||
# full-cache path shares state pages with contiguous KV pages, so
|
||||
# padding would break page matching.
|
||||
is_flashmla = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576, # NOTE: FlashMLA requires 576B alignment
|
||||
alignment=576 if is_flashmla else None,
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
@@ -333,26 +337,40 @@ class DeepseekCompressor(nn.Module):
|
||||
# - position used: (positions // compress_ratio) * compress_ratio
|
||||
cos_sin_cache = rotary_emb.cos_sin_cache
|
||||
k_cache_metadata = cast(Any, attn_metadata[self.k_cache_prefix])
|
||||
kv_cache = self._static_forward_context[self.k_cache_prefix].kv_cache
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# NVIDIA GPUs.
|
||||
if self.head_dim == 512:
|
||||
from .nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
# FlashInfer V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# legacy FlashMLA path uses the UE8M0 paged uint8 layout.
|
||||
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
|
||||
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
|
||||
fp8_scale = (
|
||||
getattr(k_cache_layer, "_flashinfer_fp8_kv_scale", None)
|
||||
if store_full_fp8
|
||||
else None
|
||||
)
|
||||
|
||||
# Main compressor path.
|
||||
# Use a cutedsl kernel for better performance.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
else:
|
||||
# Indexer path (head_dim == 128).
|
||||
# Use a triton kernel.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
# cutedsl (head=512) accepts the full-cache flags; triton (indexer/AMD)
|
||||
# does not, so the two callables have different signatures.
|
||||
compress_norm_rope_store_fn: Any
|
||||
if current_platform.is_cuda() and self.head_dim == 512:
|
||||
from .nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
|
||||
# head=512 on CUDA always uses cutedsl, for both the legacy UE8M0
|
||||
# layout and the FlashInfer full-cache layout. The full-cache flags
|
||||
# are consumed only here.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
extra_kwargs: dict[str, Any] = dict(
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
else:
|
||||
# AMD GPUs.
|
||||
# Always use a triton kernel.
|
||||
# Indexer path (head_dim == 128) or AMD: triton, legacy UE8M0 only.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_triton
|
||||
extra_kwargs = {}
|
||||
|
||||
compress_norm_rope_store_fn(
|
||||
state_cache=state_cache,
|
||||
@@ -377,4 +395,5 @@ class DeepseekCompressor(nn.Module):
|
||||
quant_block=self._quant_block,
|
||||
token_stride=self._token_stride,
|
||||
scale_dim=self._scale_dim,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4 FlashInfer TRTLLM-gen sparse MLA backend.
|
||||
|
||||
Uses FlashInfer's public ``trtllm_batch_decode_sparse_mla_dsv4`` launcher with a
|
||||
contiguous bf16 / per-tensor FP8 KV cache. Shares the V4 sparse-index pipeline
|
||||
(SWA cache + compressor + indexer, 256-token blocks, head_size 512) with the
|
||||
FlashMLA V4 backend; only the attention forward differs.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import (
|
||||
DeepseekV4FlashMLASparseBackend,
|
||||
DeepseekV4SparseMLAAttentionImpl,
|
||||
)
|
||||
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseMetadata
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4MLAAttention
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
# 128 MB TRTLLM-gen workspace, allocated once per device and zero-initialized
|
||||
# (required for first use). Reused across all FlashInfer V4 layers.
|
||||
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
_flashinfer_dsv4_workspace_by_device: dict[torch.device, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_flashinfer_dsv4_workspace(device: torch.device) -> torch.Tensor:
|
||||
workspace = _flashinfer_dsv4_workspace_by_device.get(device)
|
||||
if workspace is None:
|
||||
workspace = torch.zeros(
|
||||
_FLASHINFER_DSV4_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
_flashinfer_dsv4_workspace_by_device[device] = workspace
|
||||
return workspace
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLASparseBackend):
|
||||
"""Shares the FlashMLA V4 metadata/cache pipeline; swaps the attention impl.
|
||||
|
||||
Inheriting from the FlashMLA V4 backend reuses its ``FlashMLASparseMetadata``
|
||||
builder (which the V4 sparse-index pipeline needs — the V3.2 FlashInfer
|
||||
builder lacks the ``c128a_*`` fields), 256-token blocks, head_size 512, and
|
||||
the contiguous (num_blocks, block_size, 512) cache shape for non-``fp8_ds_mla``
|
||||
dtypes.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHINFER_MLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["DeepseekV4FlashInferMLASparseImpl"]:
|
||||
return DeepseekV4FlashInferMLASparseImpl
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseImpl(DeepseekV4SparseMLAAttentionImpl):
|
||||
"""FlashInfer TRTLLM-gen sparse MLA implementation for DeepSeek V4."""
|
||||
|
||||
backend_cls = DeepseekV4FlashInferMLASparseBackend
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# FP8 decode kernel only supports h_q = 64 or 128.
|
||||
if num_heads > 128:
|
||||
raise ValueError(
|
||||
f"DeepseekV4 Flashinfer MLA Sparse does not support {num_heads} heads "
|
||||
"(FP8 decode kernel requires h_q in {64, 128})."
|
||||
)
|
||||
return 64 if num_heads <= 64 else 128
|
||||
|
||||
@classmethod
|
||||
def init_layer_buffers(cls, layer: "DeepseekV4MLAAttention") -> None:
|
||||
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
|
||||
# per-tensor FP8 cache path consumes these; bf16 reads ``layer.scale``.
|
||||
if layer.kv_cache_torch_dtype != torch.float8_e4m3fn:
|
||||
return
|
||||
# TODO: load real per-tensor Q/KV scales from the checkpoint; unit
|
||||
# scales until the scale tensor names are wired.
|
||||
fp8_q_scale = 1.0
|
||||
fp8_kv_scale = 1.0
|
||||
layer.register_buffer(
|
||||
"_flashinfer_fp8_q_scale",
|
||||
torch.tensor([fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
layer.register_buffer(
|
||||
"_flashinfer_fp8_q_scale_inv",
|
||||
torch.tensor([1.0 / fp8_q_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
layer.register_buffer(
|
||||
"_flashinfer_fp8_kv_scale",
|
||||
torch.tensor([fp8_kv_scale], dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
# TRTLLM-gen takes scalar scale args on a distinct (correct) C++ path
|
||||
# vs 1-elem tensors, so these are Python floats. bmm1 folds the softmax
|
||||
# scale and the Q/KV per-tensor scales; bmm2 is the KV scale.
|
||||
layer._flashinfer_fp8_bmm1_scale = layer.scale * fp8_q_scale * fp8_kv_scale
|
||||
layer._flashinfer_fp8_bmm2_scale = fp8_kv_scale
|
||||
|
||||
@classmethod
|
||||
def forward_mqa( # type: ignore[override]
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
# The TRTLLM-gen kernel requires h_q in {64, 128}, so the output buffer
|
||||
# is allocated at the padded head count while q arrives at the local
|
||||
# head count; _forward pads q to match before the launcher.
|
||||
assert output.shape[0] == q.shape[0] and output.shape[-1] == q.shape[-1], (
|
||||
f"output buffer shape {output.shape} incompatible with q shape {q.shape}"
|
||||
)
|
||||
assert output.shape[1] >= q.shape[1], (
|
||||
f"output heads {output.shape[1]} must be >= q heads {q.shape[1]}"
|
||||
)
|
||||
# Per-tensor FP8 q produces a bf16 attention output.
|
||||
expected_output_dtype = (
|
||||
torch.bfloat16 if q.dtype == torch.float8_e4m3fn else q.dtype
|
||||
)
|
||||
assert output.dtype == expected_output_dtype, (
|
||||
f"output dtype {output.dtype} must match expected {expected_output_dtype} "
|
||||
f"for q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: FlashInfer reads the cache directly and lazily
|
||||
# allocates its workspace, so nothing to reserve here.
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
flashmla_metadata = cast(
|
||||
FlashMLASparseMetadata | None, attn_metadata.get(layer.prefix)
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
attn_metadata.get(layer.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = layer.compress_ratio <= 1
|
||||
# SWA-only layers don't allocate their own compressed KV cache.
|
||||
self_kv_cache = layer.kv_cache if not swa_only else None
|
||||
swa_kv_cache = layer.swa_cache_layer.kv_cache
|
||||
|
||||
cls._forward(
|
||||
layer=layer,
|
||||
q=q,
|
||||
kv_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=flashmla_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _build_sparse_index_metadata(
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: FlashMLASparseMetadata | None,
|
||||
swa_only: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build the combined sparse-index tensors for the mixed batch.
|
||||
|
||||
Returns ``(compressed_kv_cache, seq_lens, sparse_indices,
|
||||
sparse_topk_lens)``.
|
||||
"""
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_reqs = num_decodes + num_prefills
|
||||
num_tokens = num_decode_tokens + num_prefill_tokens
|
||||
|
||||
assert swa_metadata.seq_lens is not None
|
||||
assert swa_metadata.query_start_loc is not None
|
||||
assert swa_metadata.token_to_req_indices is not None
|
||||
assert swa_metadata.decode_swa_indices is not None
|
||||
assert swa_metadata.block_table is not None
|
||||
|
||||
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
|
||||
num_decode_tokens, layer.window_size
|
||||
)
|
||||
decode_compressed_topk_lens = None
|
||||
decode_compressed_indices_are_local = False
|
||||
decode_is_valid_token = None
|
||||
|
||||
if swa_only:
|
||||
assert layer.topk_indices_buffer is not None
|
||||
compressed_kv_cache = swa_k_cache
|
||||
decode_compressed_indices = None
|
||||
prefill_topk_indices = layer.topk_indices_buffer[
|
||||
num_decode_tokens:num_tokens, :0
|
||||
]
|
||||
compressed_block_table = None
|
||||
compressed_block_size = swa_metadata.block_size
|
||||
top_k = 0
|
||||
else:
|
||||
assert kv_cache is not None
|
||||
assert attn_metadata is not None
|
||||
compressed_kv_cache = kv_cache
|
||||
compressed_block_table = attn_metadata.block_table[:num_reqs]
|
||||
compressed_block_size = attn_metadata.block_size // layer.compress_ratio
|
||||
|
||||
if layer.compress_ratio == 4:
|
||||
assert layer.topk_indices_buffer is not None
|
||||
if num_prefill_tokens > 0:
|
||||
prefill_topk_indices = layer.topk_indices_buffer[
|
||||
num_decode_tokens:num_tokens
|
||||
]
|
||||
top_k = prefill_topk_indices.shape[-1]
|
||||
else:
|
||||
prefill_topk_indices = layer.topk_indices_buffer[:0, :0]
|
||||
top_k = 0
|
||||
|
||||
decode_compressed_indices_are_local = True
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
decode_is_valid_token = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if num_decode_tokens > 0:
|
||||
decode_compressed_indices = layer.topk_indices_buffer[
|
||||
:num_decode_tokens
|
||||
]
|
||||
else:
|
||||
# Keep the logical width aligned with the mixed-batch case so
|
||||
# pure-prefill steps reuse the same Triton specialization.
|
||||
decode_compressed_indices = prefill_topk_indices[:0]
|
||||
else:
|
||||
if num_prefill_tokens > 0:
|
||||
assert attn_metadata.c128a_prefill_topk_indices is not None
|
||||
prefill_topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
top_k = prefill_topk_indices.shape[-1]
|
||||
else:
|
||||
prefill_topk_indices = decode_swa_indices[:0, :0]
|
||||
top_k = 0
|
||||
|
||||
if num_decode_tokens > 0:
|
||||
assert attn_metadata.c128a_global_decode_topk_indices is not None
|
||||
assert attn_metadata.c128a_decode_topk_lens is not None
|
||||
decode_compressed_indices = (
|
||||
attn_metadata.c128a_global_decode_topk_indices.view(
|
||||
num_decode_tokens, -1
|
||||
)
|
||||
)
|
||||
decode_compressed_topk_lens = attn_metadata.c128a_decode_topk_lens
|
||||
if num_prefill_tokens == 0:
|
||||
prefill_topk_indices = decode_compressed_indices[:0, :0]
|
||||
else:
|
||||
decode_compressed_indices = prefill_topk_indices[:0]
|
||||
decode_compressed_topk_lens = swa_metadata.seq_lens[:0]
|
||||
|
||||
query_start_loc = swa_metadata.query_start_loc[: num_reqs + 1]
|
||||
seq_lens = swa_metadata.seq_lens[:num_reqs]
|
||||
assert seq_lens.dtype == torch.int32
|
||||
sparse_indices, sparse_topk_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices,
|
||||
decode_compressed_indices,
|
||||
decode_compressed_topk_lens,
|
||||
prefill_topk_indices[:num_prefill_tokens],
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
swa_metadata.token_to_req_indices[:num_tokens],
|
||||
swa_metadata.block_table[:num_reqs],
|
||||
swa_metadata.block_size,
|
||||
compressed_block_table,
|
||||
compressed_block_size,
|
||||
layer.window_size,
|
||||
layer.compress_ratio,
|
||||
top_k,
|
||||
decode_compressed_indices_are_local=decode_compressed_indices_are_local,
|
||||
decode_is_valid_token=decode_is_valid_token,
|
||||
)
|
||||
return compressed_kv_cache, seq_lens, sparse_indices, sparse_topk_lens
|
||||
|
||||
@classmethod
|
||||
def _forward(
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
attn_metadata: FlashMLASparseMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert layer.kv_cache_torch_dtype in (torch.bfloat16, torch.float8_e4m3fn)
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_reqs = num_decodes + num_prefills
|
||||
num_tokens = num_decode_tokens + num_prefill_tokens
|
||||
if num_tokens == 0:
|
||||
return
|
||||
|
||||
(
|
||||
compressed_kv_cache,
|
||||
seq_lens,
|
||||
sparse_indices,
|
||||
sparse_topk_lens,
|
||||
) = cls._build_sparse_index_metadata(
|
||||
layer=layer,
|
||||
kv_cache=kv_cache,
|
||||
swa_k_cache=swa_k_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=attn_metadata,
|
||||
swa_only=swa_only,
|
||||
)
|
||||
|
||||
# CUDA graph execution can pad q/output past the scheduled token count;
|
||||
# restrict to the real tokens (the launcher validates sparse indices).
|
||||
query = q[:num_tokens]
|
||||
output = output[:num_tokens]
|
||||
bmm1_scale: float | torch.Tensor = layer.scale
|
||||
bmm2_scale: float | torch.Tensor = 1.0
|
||||
if layer.kv_cache_torch_dtype == torch.float8_e4m3fn:
|
||||
assert query.dtype == torch.float8_e4m3fn
|
||||
bmm1_scale = layer._flashinfer_fp8_bmm1_scale
|
||||
bmm2_scale = layer._flashinfer_fp8_bmm2_scale
|
||||
else:
|
||||
assert query.dtype == torch.bfloat16
|
||||
query = query.contiguous()
|
||||
|
||||
# The TRTLLM-gen sparse-MLA kernel requires h_q in {64, 128}; zero-pad
|
||||
# the query heads to the allocated output head count. Padded heads attend
|
||||
# to the shared KV and are sliced off downstream (output is padded too).
|
||||
padded_heads = output.shape[1]
|
||||
if query.shape[1] < padded_heads:
|
||||
padded_query = query.new_zeros(
|
||||
(query.shape[0], padded_heads, query.shape[2])
|
||||
)
|
||||
padded_query[:, : query.shape[1], :] = query
|
||||
query = padded_query
|
||||
|
||||
workspace = _get_flashinfer_dsv4_workspace(q.device)
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
assert query_start_loc is not None and query_start_loc_cpu is not None
|
||||
|
||||
# Keep Perkz's two-call decode/prefill split: the TRTLLM-gen launcher is
|
||||
# tuned for uniform-q batches, and collapsing the mixed batch into a
|
||||
# single call is the suspected source of the prior IMA.
|
||||
if num_decode_tokens > 0:
|
||||
decode_cu = query_start_loc[: num_decodes + 1]
|
||||
decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1]
|
||||
decode_lens_cpu = decode_cu_cpu[1:] - decode_cu_cpu[:-1]
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=query[:num_decode_tokens],
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[:num_decode_tokens],
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[:num_decode_tokens],
|
||||
seq_lens=seq_lens[:num_decodes],
|
||||
out=output[:num_decode_tokens],
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=layer.attn_sink,
|
||||
cum_seq_lens_q=decode_cu,
|
||||
max_q_len=int(decode_lens_cpu.max().item()),
|
||||
)
|
||||
|
||||
if num_prefill_tokens > 0:
|
||||
# The prefill query view re-anchors at offset 0, so rebase the
|
||||
# cumulative query offsets to start at 0.
|
||||
prefill_cu = (
|
||||
query_start_loc[num_decodes : num_reqs + 1]
|
||||
- query_start_loc[num_decodes]
|
||||
)
|
||||
prefill_cu_cpu = query_start_loc_cpu[num_decodes : num_reqs + 1]
|
||||
prefill_lens_cpu = prefill_cu_cpu[1:] - prefill_cu_cpu[:-1]
|
||||
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
|
||||
query=query[num_decode_tokens:num_tokens],
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[num_decode_tokens:num_tokens],
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[num_decode_tokens:num_tokens],
|
||||
seq_lens=seq_lens[num_decodes:num_reqs],
|
||||
out=output[num_decode_tokens:num_tokens],
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=bmm2_scale,
|
||||
sinks=layer.attn_sink,
|
||||
cum_seq_lens_q=prefill_cu,
|
||||
max_q_len=int(prefill_lens_cpu.max().item()),
|
||||
)
|
||||
@@ -75,6 +75,15 @@ class DeepseekV4SparseMLAAttentionImpl(SparseMLAAttentionImpl[FlashMLASparseMeta
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def init_layer_buffers(cls, layer: "DeepseekV4MLAAttention") -> None:
|
||||
"""Register impl-specific buffers on the layer at construction.
|
||||
|
||||
No-op by default; FlashInfer overrides this to register its per-tensor
|
||||
FP8 scale buffers.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
class DeepseekV4FlashMLASparseBackend(FlashMLASparseBackend):
|
||||
@staticmethod
|
||||
@@ -83,7 +92,7 @@ class DeepseekV4FlashMLASparseBackend(FlashMLASparseBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "V4_FLASHMLA_SPARSE"
|
||||
return "FLASHMLA_SPARSE_DSV4"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["DeepseekV4SparseMLAAttentionImpl"]:
|
||||
|
||||
@@ -508,6 +508,426 @@ class SparseAttnCompressNormRopeStoreC4Kernel:
|
||||
)
|
||||
|
||||
|
||||
class SparseAttnCompressNormRopeStoreFullC4Kernel(
|
||||
SparseAttnCompressNormRopeStoreC4Kernel
|
||||
):
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
state_width: int,
|
||||
rope_head_dim: int,
|
||||
fp8_max: float,
|
||||
quant_block: int,
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
compress_ratio: int,
|
||||
overlap: bool,
|
||||
store_full_fp8: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
head_size,
|
||||
state_width,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
compress_ratio,
|
||||
overlap,
|
||||
)
|
||||
self.store_full_fp8 = store_full_fp8
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
state_cache: cute.Tensor,
|
||||
token_to_req_indices: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
block_table: cute.Tensor,
|
||||
block_size: Int64,
|
||||
rms_norm_weight: cute.Tensor,
|
||||
rms_norm_eps: Float32,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
fp8_scale: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (slot_mapping.shape[0], 1, 1)
|
||||
self.kernel(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
rms_norm_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
fp8_scale,
|
||||
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
state_cache: cute.Tensor,
|
||||
token_to_req_indices: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
block_table: cute.Tensor,
|
||||
block_size: Int64,
|
||||
rms_norm_weight: cute.Tensor,
|
||||
rms_norm_eps: Float32,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
fp8_scale: cute.Tensor,
|
||||
):
|
||||
token_idx, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
group_lane = lane_id % self.lanes_per_group
|
||||
group_idx = warp_id * self.groups_per_warp + lane_id // self.lanes_per_group
|
||||
elem_base = group_idx * self.quant_block + group_lane * self.elems_per_lane
|
||||
|
||||
slot_id = slot_mapping[token_idx]
|
||||
has_position = token_idx < positions.shape[0]
|
||||
position = Int64(0)
|
||||
if has_position:
|
||||
position = positions[token_idx]
|
||||
boundary = has_position and (
|
||||
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
|
||||
)
|
||||
has_req_idx = token_idx < token_to_req_indices.shape[0]
|
||||
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
|
||||
kv_slot_idx = Int64(-1)
|
||||
if has_kv_slot_idx:
|
||||
kv_slot_idx = kv_slot_mapping[token_idx]
|
||||
active = (
|
||||
slot_id >= Int64(0) and has_req_idx and boundary and kv_slot_idx >= Int64(0)
|
||||
)
|
||||
|
||||
if active:
|
||||
req_idx = token_to_req_indices[token_idx]
|
||||
start = position - Int64(self.window - 1)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
s_block_numbers = smem.allocate_tensor(
|
||||
Int32, cute.make_layout((self.window,)), byte_alignment=4
|
||||
)
|
||||
partial_sums = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((self.num_warps,)), byte_alignment=4
|
||||
)
|
||||
rrms_shared = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((1,)), byte_alignment=4
|
||||
)
|
||||
|
||||
for row in cutlass.range_constexpr(self.window):
|
||||
pos = start + Int64(row)
|
||||
if tid == row:
|
||||
block_number_i32 = Int32(0)
|
||||
if pos >= Int64(0):
|
||||
block_index = pos // block_size
|
||||
block_number_i32 = block_table[req_idx, block_index]
|
||||
s_block_numbers[row] = block_number_i32
|
||||
cute.arch.sync_threads()
|
||||
|
||||
local_max = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
local_sum = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
local_product = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
local_max[e] = -Float32.inf
|
||||
local_sum[e] = Float32(0.0)
|
||||
local_product[e] = Float32(0.0)
|
||||
|
||||
cp_f32x4 = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128
|
||||
)
|
||||
copy_layout = cute.make_layout(
|
||||
(self.copy_chunks, self.copy_elems),
|
||||
stride=(self.copy_elems, 1),
|
||||
)
|
||||
kv_vals = cute.make_rmem_tensor(copy_layout, Float32)
|
||||
score_vals = cute.make_rmem_tensor(copy_layout, Float32)
|
||||
|
||||
for row in cutlass.range_constexpr(self.window):
|
||||
pos = start + Int64(row)
|
||||
if pos >= Int64(0):
|
||||
block_index = pos // block_size
|
||||
block_offset = pos - block_index * block_size
|
||||
block_number = s_block_numbers[row].to(Int64)
|
||||
head_offset = Int64((row // self.compress_ratio) * self.head_dim)
|
||||
row_tensor = state_cache[block_number, block_offset, None]
|
||||
for chunk in cutlass.range_constexpr(self.copy_chunks):
|
||||
copy_elem = const_expr(chunk * self.copy_elems)
|
||||
col_tile = (
|
||||
head_offset + (elem_base + Int32(copy_elem)).to(Int64)
|
||||
) // Int64(self.copy_elems)
|
||||
kv_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.copy_elems,),
|
||||
coord=(col_tile,),
|
||||
)
|
||||
score_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.copy_elems,),
|
||||
coord=(
|
||||
col_tile + Int64(self.state_width // self.copy_elems),
|
||||
),
|
||||
)
|
||||
cute.copy(cp_f32x4, kv_src, kv_vals[chunk, None])
|
||||
cute.copy(cp_f32x4, score_src, score_vals[chunk, None])
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
chunk = const_expr(e // self.copy_elems)
|
||||
copy_elem = const_expr(e % self.copy_elems)
|
||||
score = score_vals[chunk, copy_elem]
|
||||
kv = kv_vals[chunk, copy_elem]
|
||||
new_max = cute.arch.fmax(local_max[e], score)
|
||||
old_scale = cute.math.exp2(
|
||||
(local_max[e] - new_max) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
new_scale = cute.math.exp2(
|
||||
(score - new_max) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
local_sum[e] = local_sum[e] * old_scale + new_scale
|
||||
local_product[e] = local_product[e] * old_scale + kv * new_scale
|
||||
local_max[e] = new_max
|
||||
|
||||
x = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
local_sumsq = Float32(0.0)
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
x[e] = local_product[e] / local_sum[e]
|
||||
local_sumsq += x[e] * x[e]
|
||||
|
||||
warp_sum = local_sumsq
|
||||
for step in cutlass.range_constexpr(5):
|
||||
offset = const_expr(16 >> step)
|
||||
warp_sum += cute.arch.shuffle_sync_bfly(warp_sum, offset)
|
||||
|
||||
if lane_id == 0:
|
||||
partial_sums[warp_id] = warp_sum
|
||||
cute.arch.sync_threads()
|
||||
if tid == 0:
|
||||
total = Float32(0.0)
|
||||
for i in cutlass.range_constexpr(self.num_warps):
|
||||
total += partial_sums[i]
|
||||
rrms_shared[0] = cute.math.rsqrt(
|
||||
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
rrms = rrms_shared[0]
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
elem = elem_base + e
|
||||
x[e] = x[e] * rrms * rms_norm_weight[elem].to(Float32)
|
||||
|
||||
page = kv_slot_idx // kv_cache_block_size
|
||||
kv_offset = kv_slot_idx - page * kv_cache_block_size
|
||||
value_base = page * k_cache.stride[0] + kv_offset * k_cache.stride[1]
|
||||
|
||||
if const_expr(self.store_full_fp8):
|
||||
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
|
||||
inv_fp8 = Float32(1.0) / fp8_scale[0]
|
||||
if group_idx == self.nope_blocks:
|
||||
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
|
||||
self.compress_ratio
|
||||
)
|
||||
for pair in cutlass.range_constexpr(self.elems_per_lane // 2):
|
||||
elem = const_expr(pair * 2)
|
||||
pair_idx = (elem_base - self.nope_dim) // 2 + Int32(pair)
|
||||
cos_v = cos_sin_cache[compressed_pos, pair_idx]
|
||||
sin_v = cos_sin_cache[
|
||||
compressed_pos, pair_idx + Int32(self.rope_dim // 2)
|
||||
]
|
||||
real = x[elem] * cos_v - x[elem + 1] * sin_v
|
||||
imag = x[elem] * sin_v + x[elem + 1] * cos_v
|
||||
packed_bf16 = _fp32x2_to_bf16x2(real, imag)
|
||||
b0, b1 = _bf16x2_to_fp32(packed_bf16)
|
||||
y0 = cutlass.min(
|
||||
cutlass.max(b0 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
y1 = cutlass.min(
|
||||
cutlass.max(b1 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
packed_fp8 = _fp32x2_to_fp8e4m3x2(y0, y1)
|
||||
out_base = value_base + (elem_base + Int32(elem)).to(Int64)
|
||||
k_cache_u16.iterator[out_base // Int64(2)] = packed_fp8
|
||||
else:
|
||||
for pair in cutlass.range_constexpr(self.elems_per_lane // 2):
|
||||
elem = const_expr(pair * 2)
|
||||
packed_bf16 = _fp32x2_to_bf16x2(x[elem], x[elem + 1])
|
||||
b0, b1 = _bf16x2_to_fp32(packed_bf16)
|
||||
y0 = cutlass.min(
|
||||
cutlass.max(b0 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
y1 = cutlass.min(
|
||||
cutlass.max(b1 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
packed_fp8 = _fp32x2_to_fp8e4m3x2(y0, y1)
|
||||
out_base = value_base + (elem_base + Int32(elem)).to(Int64)
|
||||
k_cache_u16.iterator[out_base // Int64(2)] = packed_fp8
|
||||
else:
|
||||
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
|
||||
if group_idx == self.nope_blocks:
|
||||
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
|
||||
self.compress_ratio
|
||||
)
|
||||
for pair in cutlass.range_constexpr(self.elems_per_lane // 2):
|
||||
elem = const_expr(pair * 2)
|
||||
pair_idx = (elem_base - self.nope_dim) // 2 + Int32(pair)
|
||||
cos_v = cos_sin_cache[compressed_pos, pair_idx]
|
||||
sin_v = cos_sin_cache[
|
||||
compressed_pos, pair_idx + Int32(self.rope_dim // 2)
|
||||
]
|
||||
real = x[elem] * cos_v - x[elem + 1] * sin_v
|
||||
imag = x[elem] * sin_v + x[elem + 1] * cos_v
|
||||
packed_bf16 = _fp32x2_to_bf16x2(real, imag)
|
||||
out_base = value_base + ((elem_base + Int32(elem)) * 2).to(
|
||||
Int64
|
||||
)
|
||||
k_cache_u32.iterator[out_base // Int64(4)] = packed_bf16
|
||||
else:
|
||||
for pair in cutlass.range_constexpr(self.elems_per_lane // 2):
|
||||
elem = const_expr(pair * 2)
|
||||
packed_bf16 = _fp32x2_to_bf16x2(x[elem], x[elem + 1])
|
||||
out_base = value_base + ((elem_base + Int32(elem)) * 2).to(
|
||||
Int64
|
||||
)
|
||||
k_cache_u32.iterator[out_base // Int64(4)] = packed_bf16
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
head_size: int = 512,
|
||||
state_width: int = 1024,
|
||||
rope_head_dim: int = 64,
|
||||
fp8_max: float = 448.0,
|
||||
quant_block: int = 64,
|
||||
token_stride: int = 576,
|
||||
scale_dim: int = 8,
|
||||
kv_block_stride: int = 74752,
|
||||
compress_ratio: int = 4,
|
||||
overlap: bool = True,
|
||||
store_full_fp8: bool = False,
|
||||
norm_weight_dtype: type[cutlass.Numeric] = Float32,
|
||||
):
|
||||
if compress_ratio != 4 or not overlap:
|
||||
raise ValueError("CuTe DSL C4 fused sparse-attn requires C4 overlap.")
|
||||
if head_size != 512:
|
||||
raise ValueError(
|
||||
"CuTe DSL C4 fused sparse-attn currently requires head_size=512."
|
||||
)
|
||||
if state_width != 2 * head_size:
|
||||
raise ValueError(
|
||||
"CuTe DSL C4 fused sparse-attn requires state_width=2*head_size."
|
||||
)
|
||||
if quant_block != 64:
|
||||
raise ValueError(
|
||||
"CuTe DSL C4 fused sparse-attn currently requires quant_block=64."
|
||||
)
|
||||
if rope_head_dim != 64:
|
||||
raise ValueError(
|
||||
"CuTe DSL C4 fused sparse-attn currently requires rope_head_dim=64."
|
||||
)
|
||||
num_positions = cute.sym_int()
|
||||
num_slots = cute.sym_int()
|
||||
num_req_indices = cute.sym_int()
|
||||
num_kv_slots = cute.sym_int()
|
||||
num_state_blocks = cute.sym_int()
|
||||
num_kv_blocks = cute.sym_int()
|
||||
state_cache_block_size = cute.sym_int()
|
||||
block_table_width = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
state_cache_width = state_width * 2
|
||||
|
||||
state_cache = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_state_blocks, state_cache_block_size, state_cache_width),
|
||||
stride=(
|
||||
cute.sym_int64(divisibility=16),
|
||||
cute.sym_int64(divisibility=16),
|
||||
1,
|
||||
),
|
||||
assumed_align=16,
|
||||
)
|
||||
token_to_req_indices = make_fake_tensor(
|
||||
Int32, (num_req_indices,), divisibility=4
|
||||
)
|
||||
positions = make_fake_tensor(Int64, (num_positions,), divisibility=8)
|
||||
slot_mapping = make_fake_tensor(Int64, (num_slots,), divisibility=8)
|
||||
block_table = make_fake_tensor(
|
||||
Int32, (cute.sym_int(), block_table_width), divisibility=1
|
||||
)
|
||||
rms_norm_weight = make_fake_tensor(
|
||||
norm_weight_dtype, (head_size,), divisibility=4
|
||||
)
|
||||
cos_sin_cache = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(max_pos, rope_head_dim),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
k_cache = cute.runtime.make_fake_tensor(
|
||||
Uint8,
|
||||
(num_kv_blocks, cute.sym_int(), cute.sym_int()),
|
||||
stride=(
|
||||
cute.sym_int64(divisibility=16),
|
||||
cute.sym_int64(divisibility=8),
|
||||
1,
|
||||
),
|
||||
assumed_align=16,
|
||||
)
|
||||
kv_slot_mapping = make_fake_tensor(Int64, (num_kv_slots,), divisibility=8)
|
||||
fp8_scale = make_fake_tensor(Float32, (1,), divisibility=1)
|
||||
|
||||
kernel = SparseAttnCompressNormRopeStoreFullC4Kernel(
|
||||
head_size,
|
||||
state_width,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
compress_ratio,
|
||||
overlap,
|
||||
store_full_fp8,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
Int64(0),
|
||||
rms_norm_weight,
|
||||
Float32(0.0),
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
Int64(0),
|
||||
fp8_scale,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
class SparseAttnCompressC128Block8Kernel:
|
||||
head_tile = 64
|
||||
rows_per_warp = 16
|
||||
@@ -1109,6 +1529,273 @@ class SparseAttnNormRopeStoreKernel:
|
||||
)
|
||||
|
||||
|
||||
class SparseAttnNormRopeStoreFullKernel:
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rope_head_dim: int,
|
||||
fp8_max: float,
|
||||
quant_block: int,
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
compress_ratio: int,
|
||||
store_full_fp8: bool = False,
|
||||
):
|
||||
# Standalone (not inheriting the #44230-restructured legacy kernel):
|
||||
# set attrs directly so the full-cache C128 path is decoupled.
|
||||
self.head_dim = head_size
|
||||
self.rope_dim = rope_head_dim
|
||||
self.nope_dim = head_size - rope_head_dim
|
||||
self.fp8_max = fp8_max
|
||||
self.quant_block = quant_block
|
||||
self.token_stride = token_stride
|
||||
self.scale_dim = scale_dim
|
||||
self.num_warps = head_size // quant_block
|
||||
self.nope_blocks = self.nope_dim // quant_block
|
||||
self.tb_size = head_size // 2
|
||||
self.compress_ratio = compress_ratio
|
||||
self.store_full_fp8 = store_full_fp8
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
compressed_kv: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
rms_norm_weight: cute.Tensor,
|
||||
rms_norm_eps: Float32,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
fp8_scale: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (slot_mapping.shape[0], 1, 1)
|
||||
self.kernel(
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
rms_norm_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
fp8_scale,
|
||||
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
compressed_kv: cute.Tensor,
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
rms_norm_weight: cute.Tensor,
|
||||
rms_norm_eps: Float32,
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
fp8_scale: cute.Tensor,
|
||||
):
|
||||
token_idx, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
elem0 = tid * 2
|
||||
|
||||
slot_id = slot_mapping[token_idx]
|
||||
has_position = token_idx < positions.shape[0]
|
||||
position = Int64(0)
|
||||
if has_position:
|
||||
position = positions[token_idx]
|
||||
boundary = has_position and (
|
||||
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
|
||||
)
|
||||
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
|
||||
kv_slot_idx = Int64(-1)
|
||||
if has_kv_slot_idx:
|
||||
kv_slot_idx = kv_slot_mapping[token_idx]
|
||||
active = slot_id >= Int64(0) and boundary and kv_slot_idx >= Int64(0)
|
||||
|
||||
if active:
|
||||
base = token_idx.to(Int64) * compressed_kv.stride[0] + elem0.to(Int64)
|
||||
x0 = compressed_kv.iterator[base]
|
||||
x1 = compressed_kv.iterator[base + Int64(1)]
|
||||
|
||||
local_sumsq = x0 * x0 + x1 * x1
|
||||
warp_sum = local_sumsq
|
||||
for step in cutlass.range_constexpr(5):
|
||||
offset = const_expr(16 >> step)
|
||||
warp_sum += cute.arch.shuffle_sync_bfly(warp_sum, offset)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
partial_sums = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((self.num_warps,)), byte_alignment=4
|
||||
)
|
||||
rrms_shared = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((1,)), byte_alignment=4
|
||||
)
|
||||
|
||||
if lane_id == 0:
|
||||
partial_sums[warp_id] = warp_sum
|
||||
cute.arch.sync_threads()
|
||||
if tid == 0:
|
||||
total = Float32(0.0)
|
||||
for i in cutlass.range_constexpr(self.num_warps):
|
||||
total += partial_sums[i]
|
||||
rrms_shared[0] = cute.math.rsqrt(
|
||||
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
rrms = rrms_shared[0]
|
||||
x0 = x0 * rrms * rms_norm_weight[elem0].to(Float32)
|
||||
x1 = x1 * rrms * rms_norm_weight[elem0 + 1].to(Float32)
|
||||
|
||||
page = kv_slot_idx // kv_cache_block_size
|
||||
kv_offset = kv_slot_idx - page * kv_cache_block_size
|
||||
value_base = page * k_cache.stride[0] + kv_offset * k_cache.stride[1]
|
||||
|
||||
if const_expr(self.store_full_fp8):
|
||||
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
|
||||
inv_fp8 = Float32(1.0) / fp8_scale[0]
|
||||
fp8_v0 = x0
|
||||
fp8_v1 = x1
|
||||
if warp_id == self.nope_blocks:
|
||||
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
|
||||
self.compress_ratio
|
||||
)
|
||||
pair_idx = lane_id
|
||||
cs_base = compressed_pos * cos_sin_cache.stride[0] + pair_idx.to(
|
||||
Int64
|
||||
)
|
||||
cos_v = cos_sin_cache.iterator[cs_base]
|
||||
sin_v = cos_sin_cache.iterator[cs_base + Int64(self.rope_dim // 2)]
|
||||
fp8_v0 = x0 * cos_v - x1 * sin_v
|
||||
fp8_v1 = x0 * sin_v + x1 * cos_v
|
||||
fp8_packed_bf16 = _fp32x2_to_bf16x2(fp8_v0, fp8_v1)
|
||||
b0, b1 = _bf16x2_to_fp32(fp8_packed_bf16)
|
||||
y0 = cutlass.min(
|
||||
cutlass.max(b0 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
y1 = cutlass.min(
|
||||
cutlass.max(b1 * inv_fp8, Float32(-self.fp8_max)),
|
||||
Float32(self.fp8_max),
|
||||
)
|
||||
packed_fp8 = _fp32x2_to_fp8e4m3x2(y0, y1)
|
||||
out_base = value_base + elem0.to(Int64)
|
||||
k_cache_u16.iterator[out_base // Int64(2)] = packed_fp8
|
||||
else:
|
||||
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
|
||||
bf16_v0 = x0
|
||||
bf16_v1 = x1
|
||||
if warp_id == self.nope_blocks:
|
||||
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
|
||||
self.compress_ratio
|
||||
)
|
||||
pair_idx = lane_id
|
||||
cs_base = compressed_pos * cos_sin_cache.stride[0] + pair_idx.to(
|
||||
Int64
|
||||
)
|
||||
cos_v = cos_sin_cache.iterator[cs_base]
|
||||
sin_v = cos_sin_cache.iterator[cs_base + Int64(self.rope_dim // 2)]
|
||||
bf16_v0 = x0 * cos_v - x1 * sin_v
|
||||
bf16_v1 = x0 * sin_v + x1 * cos_v
|
||||
bf16_packed = _fp32x2_to_bf16x2(bf16_v0, bf16_v1)
|
||||
out_base = value_base + (elem0 * 2).to(Int64)
|
||||
k_cache_u32.iterator[out_base // Int64(4)] = bf16_packed
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
head_size: int = 512,
|
||||
rope_head_dim: int = 64,
|
||||
fp8_max: float = 448.0,
|
||||
quant_block: int = 64,
|
||||
token_stride: int = 576,
|
||||
scale_dim: int = 8,
|
||||
kv_block_stride: int = 74752,
|
||||
compress_ratio: int = 128,
|
||||
store_full_fp8: bool = False,
|
||||
norm_weight_dtype: type[cutlass.Numeric] = Float32,
|
||||
):
|
||||
if quant_block != 64:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store currently requires quant_block=64."
|
||||
)
|
||||
if rope_head_dim != 64:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store currently requires rope_head_dim=64."
|
||||
)
|
||||
if head_size % quant_block != 0:
|
||||
raise ValueError("head_size must be divisible by quant_block.")
|
||||
num_positions = cute.sym_int()
|
||||
num_slots = cute.sym_int()
|
||||
num_kv_slots = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
num_blocks = cute.sym_int()
|
||||
|
||||
compressed_kv = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_slots, head_size),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
positions = make_fake_tensor(Int64, (num_positions,), divisibility=8)
|
||||
slot_mapping = make_fake_tensor(Int64, (num_slots,), divisibility=8)
|
||||
rms_norm_weight = make_fake_tensor(
|
||||
norm_weight_dtype, (head_size,), divisibility=4
|
||||
)
|
||||
cos_sin_cache = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(max_pos, rope_head_dim),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
k_cache = cute.runtime.make_fake_tensor(
|
||||
Uint8,
|
||||
(num_blocks, cute.sym_int(), cute.sym_int()),
|
||||
stride=(
|
||||
cute.sym_int64(divisibility=16),
|
||||
cute.sym_int64(divisibility=8),
|
||||
1,
|
||||
),
|
||||
assumed_align=16,
|
||||
)
|
||||
kv_slot_mapping = make_fake_tensor(Int64, (num_kv_slots,), divisibility=8)
|
||||
fp8_scale = make_fake_tensor(Float32, (1,), divisibility=1)
|
||||
|
||||
kernel = SparseAttnNormRopeStoreFullKernel(
|
||||
head_size,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
compress_ratio,
|
||||
store_full_fp8,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
rms_norm_weight,
|
||||
Float32(0.0),
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
Int64(0),
|
||||
fp8_scale,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def compile_split_sparse_attn_cutedsl(
|
||||
head_size: int,
|
||||
state_width: int,
|
||||
@@ -1123,6 +1810,8 @@ def compile_split_sparse_attn_cutedsl(
|
||||
compress_ratio: int,
|
||||
overlap: bool,
|
||||
rms_norm_weight_dtype: torch.dtype,
|
||||
store_full_kv: bool = False,
|
||||
store_full_fp8: bool = False,
|
||||
):
|
||||
if not (
|
||||
head_size == 512
|
||||
@@ -1141,18 +1830,33 @@ def compile_split_sparse_attn_cutedsl(
|
||||
state_width=state_width,
|
||||
)
|
||||
norm_weight_dtype = _TORCH_TO_CUTE[rms_norm_weight_dtype]
|
||||
store = SparseAttnNormRopeStoreKernel.compile(
|
||||
head_size,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
kv_block_stride,
|
||||
compress_ratio,
|
||||
norm_weight_dtype,
|
||||
kv_cache_block_size,
|
||||
)
|
||||
if store_full_kv:
|
||||
# FlashInfer contiguous bf16/fp8 cache: standalone full-cache store.
|
||||
store = SparseAttnNormRopeStoreFullKernel.compile(
|
||||
head_size=head_size,
|
||||
rope_head_dim=rope_head_dim,
|
||||
fp8_max=fp8_max,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
kv_block_stride=kv_block_stride,
|
||||
compress_ratio=compress_ratio,
|
||||
store_full_fp8=store_full_fp8,
|
||||
norm_weight_dtype=norm_weight_dtype,
|
||||
)
|
||||
else:
|
||||
store = SparseAttnNormRopeStoreKernel.compile(
|
||||
head_size,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
kv_block_stride,
|
||||
compress_ratio,
|
||||
norm_weight_dtype,
|
||||
kv_cache_block_size,
|
||||
)
|
||||
return compress, store
|
||||
|
||||
|
||||
@@ -1180,13 +1884,16 @@ def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
scale_dim: int = 8,
|
||||
compress_ratio: int = 128,
|
||||
overlap: bool = False,
|
||||
store_full_kv: bool = False,
|
||||
store_full_fp8: bool = False,
|
||||
fp8_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if k_cache.ndim != 3:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store expects the real DeepSeek V4 "
|
||||
f"3D k_cache layout [num_blocks, block_size, 584], got ndim={k_cache.ndim}."
|
||||
)
|
||||
if kv_cache_block_size != k_cache.shape[1]:
|
||||
if not store_full_kv and kv_cache_block_size != k_cache.shape[1]:
|
||||
raise ValueError(
|
||||
"CuTe DSL split sparse-attn wrapper expected kv_cache_block_size "
|
||||
f"to match k_cache.shape[1], got {kv_cache_block_size} and "
|
||||
@@ -1199,6 +1906,8 @@ def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
"CuTe DSL sparse-attn store supports rms_norm_weight dtype "
|
||||
f"bf16/fp32, got {rms_norm_weight.dtype}."
|
||||
)
|
||||
if store_full_fp8 and not store_full_kv:
|
||||
raise ValueError("store_full_fp8 requires store_full_kv.")
|
||||
compress, store = compile_split_sparse_attn_cutedsl(
|
||||
head_size,
|
||||
state_width,
|
||||
@@ -1213,6 +1922,8 @@ def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
compress_ratio,
|
||||
overlap,
|
||||
rms_norm_weight.dtype,
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
)
|
||||
compress(
|
||||
state_cache,
|
||||
@@ -1223,6 +1934,25 @@ def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
compressed_kv,
|
||||
)
|
||||
|
||||
if store_full_kv:
|
||||
# Byte-addressed contiguous cache; block size + per-tensor scale are
|
||||
# passed at call time (not baked into compile).
|
||||
if fp8_scale is None:
|
||||
fp8_scale = torch.ones(1, dtype=torch.float32, device=k_cache.device)
|
||||
store(
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
rms_norm_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
k_cache.view(torch.uint8),
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
fp8_scale,
|
||||
)
|
||||
return
|
||||
|
||||
store(
|
||||
compressed_kv,
|
||||
positions,
|
||||
@@ -1258,6 +1988,9 @@ def fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
scale_dim: int = 8,
|
||||
compress_ratio: int = 4,
|
||||
overlap: bool = True,
|
||||
store_full_kv: bool = False,
|
||||
store_full_fp8: bool = False,
|
||||
fp8_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if positions.numel() == 0:
|
||||
return
|
||||
@@ -1272,6 +2005,43 @@ def fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
"CuTe DSL sparse-attn fused store expects the real DeepSeek V4 "
|
||||
f"3D k_cache layout [num_blocks, block_size, 584], got ndim={k_cache.ndim}."
|
||||
)
|
||||
if store_full_fp8 and not store_full_kv:
|
||||
raise ValueError("store_full_fp8 requires store_full_kv.")
|
||||
if store_full_kv:
|
||||
# FlashInfer contiguous bf16/fp8 cache: byte-addressed full-cache C4 store.
|
||||
if fp8_scale is None:
|
||||
fp8_scale = torch.ones(1, dtype=torch.float32, device=k_cache.device)
|
||||
compiled = SparseAttnCompressNormRopeStoreFullC4Kernel.compile(
|
||||
head_size=head_size,
|
||||
state_width=state_width,
|
||||
rope_head_dim=rope_head_dim,
|
||||
fp8_max=fp8_max,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
kv_block_stride=kv_block_stride,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
store_full_fp8=store_full_fp8,
|
||||
norm_weight_dtype=norm_weight_dtype,
|
||||
)
|
||||
compiled(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
rms_norm_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
k_cache.view(torch.uint8),
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
fp8_scale,
|
||||
)
|
||||
return
|
||||
|
||||
compiled = SparseAttnCompressNormRopeStoreC4Kernel.compile(
|
||||
head_size=head_size,
|
||||
state_width=state_width,
|
||||
@@ -1324,6 +2094,9 @@ def compress_norm_rope_store_cutedsl(
|
||||
quant_block: int,
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
store_full_kv: bool = False,
|
||||
store_full_fp8: bool = False,
|
||||
fp8_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if compress_ratio == 4:
|
||||
# For C4A, the single fused kernel is faster than the two-kernel version.
|
||||
@@ -1350,6 +2123,9 @@ def compress_norm_rope_store_cutedsl(
|
||||
scale_dim=scale_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
else:
|
||||
# For C128, the two-kernel version is faster than the single fused kernel.
|
||||
@@ -1382,4 +2158,7 @@ def compress_norm_rope_store_cutedsl(
|
||||
scale_dim=scale_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
store_full_kv=store_full_kv,
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user